* [CI 1/6] drm/i915: Introduce concept of per-timeline (context) HWSP
@ 2019-01-28 18:18 Chris Wilson
2019-01-28 18:18 ` [CI 2/6] drm/i915: Enlarge vma->pin_count Chris Wilson
` (8 more replies)
0 siblings, 9 replies; 10+ messages in thread
From: Chris Wilson @ 2019-01-28 18:18 UTC (permalink / raw)
To: intel-gfx
Supplement the per-engine HWSP with a per-timeline HWSP. That is a
per-request pointer through which we can check a local seqno,
abstracting away the presumption of a global seqno. In this first step,
we point each request back into the engine's HWSP so everything
continues to work with the global timeline.
v2: s/i915_request_hwsp/hwsp_seqno/ to emphasis that this is the current
HW value and that we are accessing it via i915_request merely as a
convenience.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Reviewed-by: John Harrison <John.C.Harrison@Intel.com>
---
drivers/gpu/drm/i915/i915_request.c | 16 ++++++----
drivers/gpu/drm/i915/i915_request.h | 45 ++++++++++++++++++++++++-----
drivers/gpu/drm/i915/intel_lrc.c | 9 ++++--
3 files changed, 55 insertions(+), 15 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index f4241a17e2ad..a076fd0b7ba6 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -182,10 +182,11 @@ static void free_capture_list(struct i915_request *request)
static void __retire_engine_request(struct intel_engine_cs *engine,
struct i915_request *rq)
{
- GEM_TRACE("%s(%s) fence %llx:%lld, global=%d, current %d\n",
+ GEM_TRACE("%s(%s) fence %llx:%lld, global=%d, current %d:%d\n",
__func__, engine->name,
rq->fence.context, rq->fence.seqno,
rq->global_seqno,
+ hwsp_seqno(rq),
intel_engine_get_seqno(engine));
GEM_BUG_ON(!i915_request_completed(rq));
@@ -244,10 +245,11 @@ static void i915_request_retire(struct i915_request *request)
{
struct i915_gem_active *active, *next;
- GEM_TRACE("%s fence %llx:%lld, global=%d, current %d\n",
+ GEM_TRACE("%s fence %llx:%lld, global=%d, current %d:%d\n",
request->engine->name,
request->fence.context, request->fence.seqno,
request->global_seqno,
+ hwsp_seqno(request),
intel_engine_get_seqno(request->engine));
lockdep_assert_held(&request->i915->drm.struct_mutex);
@@ -307,10 +309,11 @@ void i915_request_retire_upto(struct i915_request *rq)
struct intel_ring *ring = rq->ring;
struct i915_request *tmp;
- GEM_TRACE("%s fence %llx:%lld, global=%d, current %d\n",
+ GEM_TRACE("%s fence %llx:%lld, global=%d, current %d:%d\n",
rq->engine->name,
rq->fence.context, rq->fence.seqno,
rq->global_seqno,
+ hwsp_seqno(rq),
intel_engine_get_seqno(rq->engine));
lockdep_assert_held(&rq->i915->drm.struct_mutex);
@@ -355,10 +358,11 @@ void __i915_request_submit(struct i915_request *request)
struct intel_engine_cs *engine = request->engine;
u32 seqno;
- GEM_TRACE("%s fence %llx:%lld -> global=%d, current %d\n",
+ GEM_TRACE("%s fence %llx:%lld -> global=%d, current %d:%d\n",
engine->name,
request->fence.context, request->fence.seqno,
engine->timeline.seqno + 1,
+ hwsp_seqno(request),
intel_engine_get_seqno(engine));
GEM_BUG_ON(!irqs_disabled());
@@ -405,10 +409,11 @@ void __i915_request_unsubmit(struct i915_request *request)
{
struct intel_engine_cs *engine = request->engine;
- GEM_TRACE("%s fence %llx:%lld <- global=%d, current %d\n",
+ GEM_TRACE("%s fence %llx:%lld <- global=%d, current %d:%d\n",
engine->name,
request->fence.context, request->fence.seqno,
request->global_seqno,
+ hwsp_seqno(request),
intel_engine_get_seqno(engine));
GEM_BUG_ON(!irqs_disabled());
@@ -616,6 +621,7 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
rq->ring = ce->ring;
rq->timeline = ce->ring->timeline;
GEM_BUG_ON(rq->timeline == &engine->timeline);
+ rq->hwsp_seqno = &engine->status_page.addr[I915_GEM_HWS_INDEX];
spin_lock_init(&rq->lock);
dma_fence_init(&rq->fence,
diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h
index c0f084ca4f29..ade010fe6e26 100644
--- a/drivers/gpu/drm/i915/i915_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -130,6 +130,13 @@ struct i915_request {
struct i915_sched_node sched;
struct i915_dependency dep;
+ /*
+ * A convenience pointer to the current breadcrumb value stored in
+ * the HW status page (or our timeline's local equivalent). The full
+ * path would be rq->hw_context->ring->timeline->hwsp_seqno.
+ */
+ const u32 *hwsp_seqno;
+
/**
* GEM sequence number associated with this request on the
* global execution timeline. It is zero when the request is not
@@ -285,11 +292,6 @@ static inline bool i915_request_signaled(const struct i915_request *rq)
return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags);
}
-static inline bool intel_engine_has_started(struct intel_engine_cs *engine,
- u32 seqno);
-static inline bool intel_engine_has_completed(struct intel_engine_cs *engine,
- u32 seqno);
-
/**
* Returns true if seq1 is later than seq2.
*/
@@ -298,6 +300,35 @@ static inline bool i915_seqno_passed(u32 seq1, u32 seq2)
return (s32)(seq1 - seq2) >= 0;
}
+static inline u32 __hwsp_seqno(const struct i915_request *rq)
+{
+ return READ_ONCE(*rq->hwsp_seqno);
+}
+
+/**
+ * hwsp_seqno - the current breadcrumb value in the HW status page
+ * @rq: the request, to chase the relevant HW status page
+ *
+ * The emphasis in naming here is that hwsp_seqno() is not a property of the
+ * request, but an indication of the current HW state (associated with this
+ * request). Its value will change as the GPU executes more requests.
+ *
+ * Returns the current breadcrumb value in the associated HW status page (or
+ * the local timeline's equivalent) for this request. The request itself
+ * has the associated breadcrumb value of rq->fence.seqno, when the HW
+ * status page has that breadcrumb or later, this request is complete.
+ */
+static inline u32 hwsp_seqno(const struct i915_request *rq)
+{
+ u32 seqno;
+
+ rcu_read_lock(); /* the HWSP may be freed at runtime */
+ seqno = __hwsp_seqno(rq);
+ rcu_read_unlock();
+
+ return seqno;
+}
+
/**
* i915_request_started - check if the request has begun being executed
* @rq: the request
@@ -315,14 +346,14 @@ static inline bool i915_request_started(const struct i915_request *rq)
if (!seqno) /* not yet submitted to HW */
return false;
- return intel_engine_has_started(rq->engine, seqno);
+ return i915_seqno_passed(hwsp_seqno(rq), seqno - 1);
}
static inline bool
__i915_request_completed(const struct i915_request *rq, u32 seqno)
{
GEM_BUG_ON(!seqno);
- return intel_engine_has_completed(rq->engine, seqno) &&
+ return i915_seqno_passed(hwsp_seqno(rq), seqno) &&
seqno == i915_request_global_seqno(rq);
}
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 2cf99c436658..9ae7f77293a0 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -446,11 +446,12 @@ static void execlists_submit_ports(struct intel_engine_cs *engine)
desc = execlists_update_context(rq);
GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc));
- GEM_TRACE("%s in[%d]: ctx=%d.%d, global=%d (fence %llx:%lld) (current %d), prio=%d\n",
+ GEM_TRACE("%s in[%d]: ctx=%d.%d, global=%d (fence %llx:%lld) (current %d:%d), prio=%d\n",
engine->name, n,
port[n].context_id, count,
rq->global_seqno,
rq->fence.context, rq->fence.seqno,
+ hwsp_seqno(rq),
intel_engine_get_seqno(engine),
rq_prio(rq));
} else {
@@ -742,11 +743,12 @@ execlists_cancel_port_requests(struct intel_engine_execlists * const execlists)
while (num_ports-- && port_isset(port)) {
struct i915_request *rq = port_request(port);
- GEM_TRACE("%s:port%u global=%d (fence %llx:%lld), (current %d)\n",
+ GEM_TRACE("%s:port%u global=%d (fence %llx:%lld), (current %d:%d)\n",
rq->engine->name,
(unsigned int)(port - execlists->port),
rq->global_seqno,
rq->fence.context, rq->fence.seqno,
+ hwsp_seqno(rq),
intel_engine_get_seqno(rq->engine));
GEM_BUG_ON(!execlists->active);
@@ -970,12 +972,13 @@ static void process_csb(struct intel_engine_cs *engine)
EXECLISTS_ACTIVE_USER));
rq = port_unpack(port, &count);
- GEM_TRACE("%s out[0]: ctx=%d.%d, global=%d (fence %llx:%lld) (current %d), prio=%d\n",
+ GEM_TRACE("%s out[0]: ctx=%d.%d, global=%d (fence %llx:%lld) (current %d:%d), prio=%d\n",
engine->name,
port->context_id, count,
rq ? rq->global_seqno : 0,
rq ? rq->fence.context : 0,
rq ? rq->fence.seqno : 0,
+ rq ? hwsp_seqno(rq) : 0,
intel_engine_get_seqno(engine),
rq ? rq_prio(rq) : 0);
--
2.20.1
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [CI 2/6] drm/i915: Enlarge vma->pin_count
2019-01-28 18:18 [CI 1/6] drm/i915: Introduce concept of per-timeline (context) HWSP Chris Wilson
@ 2019-01-28 18:18 ` Chris Wilson
2019-01-28 18:18 ` [CI 3/6] drm/i915: Allocate a status page for each timeline Chris Wilson
` (7 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Chris Wilson @ 2019-01-28 18:18 UTC (permalink / raw)
To: intel-gfx
Previously we only accommodated having a vma pinned by a small number of
users, with the maximum being pinned for use by the display engine. As
such, we used a small bitfield only large enough to allow the vma to
be pinned twice (for back/front buffers) in each scanout plane. Keeping
the maximum permissible pin_count small allows us to quickly catch a
potential leak. However, as we want to split a 4096B page into 64
different cachelines and pin each cacheline for use by a different
timeline, we will exceed the current maximum permissible vma->pin_count
and so time has come to enlarge it.
Whilst we are here, try to pull together the similar bits:
Address/layout specification:
- bias, mappable, zone_4g: address limit specifiers
- fixed: address override, limits still apply though
- high: not strictly an address limit, but an address direction to search
Search controls:
- nonblock, nonfault, noevict
v2: Rewrite the guideline comment on bit consumption.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: John Harrison <john.C.Harrison@intel.com>
---
drivers/gpu/drm/i915/i915_gem_gtt.h | 26 ++++++++---------
drivers/gpu/drm/i915/i915_vma.h | 45 +++++++++++++++++++----------
2 files changed, 42 insertions(+), 29 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index bd679c8c56dd..03ade71b8d9a 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -642,19 +642,19 @@ int i915_gem_gtt_insert(struct i915_address_space *vm,
/* Flags used by pin/bind&friends. */
#define PIN_NONBLOCK BIT_ULL(0)
-#define PIN_MAPPABLE BIT_ULL(1)
-#define PIN_ZONE_4G BIT_ULL(2)
-#define PIN_NONFAULT BIT_ULL(3)
-#define PIN_NOEVICT BIT_ULL(4)
-
-#define PIN_MBZ BIT_ULL(5) /* I915_VMA_PIN_OVERFLOW */
-#define PIN_GLOBAL BIT_ULL(6) /* I915_VMA_GLOBAL_BIND */
-#define PIN_USER BIT_ULL(7) /* I915_VMA_LOCAL_BIND */
-#define PIN_UPDATE BIT_ULL(8)
-
-#define PIN_HIGH BIT_ULL(9)
-#define PIN_OFFSET_BIAS BIT_ULL(10)
-#define PIN_OFFSET_FIXED BIT_ULL(11)
+#define PIN_NONFAULT BIT_ULL(1)
+#define PIN_NOEVICT BIT_ULL(2)
+#define PIN_MAPPABLE BIT_ULL(3)
+#define PIN_ZONE_4G BIT_ULL(4)
+#define PIN_HIGH BIT_ULL(5)
+#define PIN_OFFSET_BIAS BIT_ULL(6)
+#define PIN_OFFSET_FIXED BIT_ULL(7)
+
+#define PIN_MBZ BIT_ULL(8) /* I915_VMA_PIN_OVERFLOW */
+#define PIN_GLOBAL BIT_ULL(9) /* I915_VMA_GLOBAL_BIND */
+#define PIN_USER BIT_ULL(10) /* I915_VMA_LOCAL_BIND */
+#define PIN_UPDATE BIT_ULL(11)
+
#define PIN_OFFSET_MASK (-I915_GTT_PAGE_SIZE)
#endif
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index 7252abc73d3e..5793abe509a2 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -71,29 +71,42 @@ struct i915_vma {
unsigned int open_count;
unsigned long flags;
/**
- * How many users have pinned this object in GTT space. The following
- * users can each hold at most one reference: pwrite/pread, execbuffer
- * (objects are not allowed multiple times for the same batchbuffer),
- * and the framebuffer code. When switching/pageflipping, the
- * framebuffer code has at most two buffers pinned per crtc.
+ * How many users have pinned this object in GTT space.
*
- * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
- * bits with absolutely no headroom. So use 4 bits.
+ * This is a tightly bound, fairly small number of users, so we
+ * stuff inside the flags field so that we can both check for overflow
+ * and detect a no-op i915_vma_pin() in a single check, while also
+ * pinning the vma.
+ *
+ * The worst case display setup would have the same vma pinned for
+ * use on each plane on each crtc, while also building the next atomic
+ * state and holding a pin for the length of the cleanup queue. In the
+ * future, the flip queue may be increased from 1.
+ * Estimated worst case: 3 [qlen] * 4 [max crtcs] * 7 [max planes] = 84
+ *
+ * For GEM, the number of concurrent users for pwrite/pread is
+ * unbounded. For execbuffer, it is currently one but will in future
+ * be extended to allow multiple clients to pin vma concurrently.
+ *
+ * We also use suballocated pages, with each suballocation claiming
+ * its own pin on the shared vma. At present, this is limited to
+ * exclusive cachelines of a single page, so a maximum of 64 possible
+ * users.
*/
-#define I915_VMA_PIN_MASK 0xf
-#define I915_VMA_PIN_OVERFLOW BIT(5)
+#define I915_VMA_PIN_MASK 0xff
+#define I915_VMA_PIN_OVERFLOW BIT(8)
/** Flags and address space this VMA is bound to */
-#define I915_VMA_GLOBAL_BIND BIT(6)
-#define I915_VMA_LOCAL_BIND BIT(7)
+#define I915_VMA_GLOBAL_BIND BIT(9)
+#define I915_VMA_LOCAL_BIND BIT(10)
#define I915_VMA_BIND_MASK (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND | I915_VMA_PIN_OVERFLOW)
-#define I915_VMA_GGTT BIT(8)
-#define I915_VMA_CAN_FENCE BIT(9)
-#define I915_VMA_CLOSED BIT(10)
-#define I915_VMA_USERFAULT_BIT 11
+#define I915_VMA_GGTT BIT(11)
+#define I915_VMA_CAN_FENCE BIT(12)
+#define I915_VMA_CLOSED BIT(13)
+#define I915_VMA_USERFAULT_BIT 14
#define I915_VMA_USERFAULT BIT(I915_VMA_USERFAULT_BIT)
-#define I915_VMA_GGTT_WRITE BIT(12)
+#define I915_VMA_GGTT_WRITE BIT(15)
unsigned int active_count;
struct rb_root active;
--
2.20.1
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [CI 3/6] drm/i915: Allocate a status page for each timeline
2019-01-28 18:18 [CI 1/6] drm/i915: Introduce concept of per-timeline (context) HWSP Chris Wilson
2019-01-28 18:18 ` [CI 2/6] drm/i915: Enlarge vma->pin_count Chris Wilson
@ 2019-01-28 18:18 ` Chris Wilson
2019-01-28 18:18 ` [CI 4/6] drm/i915: Share per-timeline HWSP using a slab suballocator Chris Wilson
` (6 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Chris Wilson @ 2019-01-28 18:18 UTC (permalink / raw)
To: intel-gfx
Allocate a page for use as a status page by a group of timelines, as we
only need a dword of storage for each (rounded up to the cacheline for
safety) we can pack multiple timelines into the same page. Each timeline
will then be able to track its own HW seqno.
v2: Reuse the common per-engine HWSP for the solitary ringbuffer
timeline, so that we do not have to emit (using per-gen specialised
vfuncs) the breadcrumb into the distinct timeline HWSP and instead can
keep on using the common MI_STORE_DWORD_INDEX. However, to maintain the
sleight-of-hand for the global/per-context seqno switchover, we will
store both temporarily (and so use a custom offset for the shared timeline
HWSP until the switch over).
v3: Keep things simple and allocate a page for each timeline, page
sharing comes next.
v4: I was caught repeating the same MI_STORE_DWORD_IMM over and over
again in selftests.
v5: And caught red handed copying create timeline + check.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
---
drivers/gpu/drm/i915/i915_timeline.c | 121 ++++++-
drivers/gpu/drm/i915/i915_timeline.h | 21 +-
drivers/gpu/drm/i915/intel_engine_cs.c | 76 ++--
drivers/gpu/drm/i915/intel_lrc.c | 22 +-
drivers/gpu/drm/i915/intel_ringbuffer.c | 10 +-
drivers/gpu/drm/i915/intel_ringbuffer.h | 6 +-
.../drm/i915/selftests/i915_live_selftests.h | 1 +
.../drm/i915/selftests/i915_mock_selftests.h | 2 +-
.../gpu/drm/i915/selftests/i915_timeline.c | 326 +++++++++++++++++-
drivers/gpu/drm/i915/selftests/mock_engine.c | 14 +-
10 files changed, 543 insertions(+), 56 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_timeline.c b/drivers/gpu/drm/i915/i915_timeline.c
index 84550f17d3df..8d5792311a8f 100644
--- a/drivers/gpu/drm/i915/i915_timeline.c
+++ b/drivers/gpu/drm/i915/i915_timeline.c
@@ -9,28 +9,78 @@
#include "i915_timeline.h"
#include "i915_syncmap.h"
-void i915_timeline_init(struct drm_i915_private *i915,
- struct i915_timeline *timeline,
- const char *name)
+static struct i915_vma *__hwsp_alloc(struct drm_i915_private *i915)
+{
+ struct drm_i915_gem_object *obj;
+ struct i915_vma *vma;
+
+ obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
+ if (IS_ERR(obj))
+ return ERR_CAST(obj);
+
+ i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
+
+ vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
+ if (IS_ERR(vma))
+ i915_gem_object_put(obj);
+
+ return vma;
+}
+
+static int hwsp_alloc(struct i915_timeline *timeline)
+{
+ struct i915_vma *vma;
+
+ vma = __hwsp_alloc(timeline->i915);
+ if (IS_ERR(vma))
+ return PTR_ERR(vma);
+
+ timeline->hwsp_ggtt = vma;
+ timeline->hwsp_offset = 0;
+
+ return 0;
+}
+
+int i915_timeline_init(struct drm_i915_private *i915,
+ struct i915_timeline *timeline,
+ const char *name,
+ struct i915_vma *global_hwsp)
{
struct i915_gt_timelines *gt = &i915->gt.timelines;
+ void *vaddr;
+ int err;
/*
* Ideally we want a set of engines on a single leaf as we expect
* to mostly be tracking synchronisation between engines. It is not
* a huge issue if this is not the case, but we may want to mitigate
* any page crossing penalties if they become an issue.
+ *
+ * Called during early_init before we know how many engines there are.
*/
BUILD_BUG_ON(KSYNCMAP < I915_NUM_ENGINES);
timeline->i915 = i915;
timeline->name = name;
+ timeline->pin_count = 0;
+
+ if (global_hwsp) {
+ timeline->hwsp_ggtt = i915_vma_get(global_hwsp);
+ timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR;
+ } else {
+ err = hwsp_alloc(timeline);
+ if (err)
+ return err;
+ }
- mutex_lock(>->mutex);
- list_add(&timeline->link, >->list);
- mutex_unlock(>->mutex);
+ vaddr = i915_gem_object_pin_map(timeline->hwsp_ggtt->obj, I915_MAP_WB);
+ if (IS_ERR(vaddr)) {
+ i915_vma_put(timeline->hwsp_ggtt);
+ return PTR_ERR(vaddr);
+ }
- /* Called during early_init before we know how many engines there are */
+ timeline->hwsp_seqno =
+ memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES);
timeline->fence_context = dma_fence_context_alloc(1);
@@ -40,6 +90,12 @@ void i915_timeline_init(struct drm_i915_private *i915,
INIT_LIST_HEAD(&timeline->requests);
i915_syncmap_init(&timeline->sync);
+
+ mutex_lock(>->mutex);
+ list_add(&timeline->link, >->list);
+ mutex_unlock(>->mutex);
+
+ return 0;
}
void i915_timelines_init(struct drm_i915_private *i915)
@@ -85,6 +141,7 @@ void i915_timeline_fini(struct i915_timeline *timeline)
{
struct i915_gt_timelines *gt = &timeline->i915->gt.timelines;
+ GEM_BUG_ON(timeline->pin_count);
GEM_BUG_ON(!list_empty(&timeline->requests));
i915_syncmap_free(&timeline->sync);
@@ -92,23 +149,69 @@ void i915_timeline_fini(struct i915_timeline *timeline)
mutex_lock(>->mutex);
list_del(&timeline->link);
mutex_unlock(>->mutex);
+
+ i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
+ i915_vma_put(timeline->hwsp_ggtt);
}
struct i915_timeline *
-i915_timeline_create(struct drm_i915_private *i915, const char *name)
+i915_timeline_create(struct drm_i915_private *i915,
+ const char *name,
+ struct i915_vma *global_hwsp)
{
struct i915_timeline *timeline;
+ int err;
timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
if (!timeline)
return ERR_PTR(-ENOMEM);
- i915_timeline_init(i915, timeline, name);
+ err = i915_timeline_init(i915, timeline, name, global_hwsp);
+ if (err) {
+ kfree(timeline);
+ return ERR_PTR(err);
+ }
+
kref_init(&timeline->kref);
return timeline;
}
+int i915_timeline_pin(struct i915_timeline *tl)
+{
+ int err;
+
+ if (tl->pin_count++)
+ return 0;
+ GEM_BUG_ON(!tl->pin_count);
+
+ err = i915_vma_pin(tl->hwsp_ggtt, 0, 0, PIN_GLOBAL | PIN_HIGH);
+ if (err)
+ goto unpin;
+
+ return 0;
+
+unpin:
+ tl->pin_count = 0;
+ return err;
+}
+
+void i915_timeline_unpin(struct i915_timeline *tl)
+{
+ GEM_BUG_ON(!tl->pin_count);
+ if (--tl->pin_count)
+ return;
+
+ /*
+ * Since this timeline is idle, all bariers upon which we were waiting
+ * must also be complete and so we can discard the last used barriers
+ * without loss of information.
+ */
+ i915_syncmap_free(&tl->sync);
+
+ __i915_vma_unpin(tl->hwsp_ggtt);
+}
+
void __i915_timeline_free(struct kref *kref)
{
struct i915_timeline *timeline =
diff --git a/drivers/gpu/drm/i915/i915_timeline.h b/drivers/gpu/drm/i915/i915_timeline.h
index 87ad2dd31c20..0c3739d53d79 100644
--- a/drivers/gpu/drm/i915/i915_timeline.h
+++ b/drivers/gpu/drm/i915/i915_timeline.h
@@ -32,6 +32,8 @@
#include "i915_syncmap.h"
#include "i915_utils.h"
+struct i915_vma;
+
struct i915_timeline {
u64 fence_context;
u32 seqno;
@@ -40,6 +42,11 @@ struct i915_timeline {
#define TIMELINE_CLIENT 0 /* default subclass */
#define TIMELINE_ENGINE 1
+ unsigned int pin_count;
+ const u32 *hwsp_seqno;
+ struct i915_vma *hwsp_ggtt;
+ u32 hwsp_offset;
+
/**
* List of breadcrumbs associated with GPU requests currently
* outstanding.
@@ -71,9 +78,10 @@ struct i915_timeline {
struct kref kref;
};
-void i915_timeline_init(struct drm_i915_private *i915,
- struct i915_timeline *tl,
- const char *name);
+int i915_timeline_init(struct drm_i915_private *i915,
+ struct i915_timeline *tl,
+ const char *name,
+ struct i915_vma *hwsp);
void i915_timeline_fini(struct i915_timeline *tl);
static inline void
@@ -96,7 +104,9 @@ i915_timeline_set_subclass(struct i915_timeline *timeline,
}
struct i915_timeline *
-i915_timeline_create(struct drm_i915_private *i915, const char *name);
+i915_timeline_create(struct drm_i915_private *i915,
+ const char *name,
+ struct i915_vma *global_hwsp);
static inline struct i915_timeline *
i915_timeline_get(struct i915_timeline *timeline)
@@ -135,6 +145,9 @@ static inline bool i915_timeline_sync_is_later(struct i915_timeline *tl,
return __i915_timeline_sync_is_later(tl, fence->context, fence->seqno);
}
+int i915_timeline_pin(struct i915_timeline *tl);
+void i915_timeline_unpin(struct i915_timeline *tl);
+
void i915_timelines_init(struct drm_i915_private *i915);
void i915_timelines_park(struct drm_i915_private *i915);
void i915_timelines_fini(struct drm_i915_private *i915);
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index 2657eb6fd914..515e87846afd 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -484,26 +484,6 @@ static void intel_engine_init_execlist(struct intel_engine_cs *engine)
execlists->queue = RB_ROOT_CACHED;
}
-/**
- * intel_engines_setup_common - setup engine state not requiring hw access
- * @engine: Engine to setup.
- *
- * Initializes @engine@ structure members shared between legacy and execlists
- * submission modes which do not require hardware access.
- *
- * Typically done early in the submission mode specific engine setup stage.
- */
-void intel_engine_setup_common(struct intel_engine_cs *engine)
-{
- i915_timeline_init(engine->i915, &engine->timeline, engine->name);
- i915_timeline_set_subclass(&engine->timeline, TIMELINE_ENGINE);
-
- intel_engine_init_execlist(engine);
- intel_engine_init_hangcheck(engine);
- intel_engine_init_batch_pool(engine);
- intel_engine_init_cmd_parser(engine);
-}
-
static void cleanup_status_page(struct intel_engine_cs *engine)
{
struct i915_vma *vma;
@@ -601,6 +581,44 @@ static int init_status_page(struct intel_engine_cs *engine)
return ret;
}
+/**
+ * intel_engines_setup_common - setup engine state not requiring hw access
+ * @engine: Engine to setup.
+ *
+ * Initializes @engine@ structure members shared between legacy and execlists
+ * submission modes which do not require hardware access.
+ *
+ * Typically done early in the submission mode specific engine setup stage.
+ */
+int intel_engine_setup_common(struct intel_engine_cs *engine)
+{
+ int err;
+
+ err = init_status_page(engine);
+ if (err)
+ return err;
+
+ err = i915_timeline_init(engine->i915,
+ &engine->timeline,
+ engine->name,
+ engine->status_page.vma);
+ if (err)
+ goto err_hwsp;
+
+ i915_timeline_set_subclass(&engine->timeline, TIMELINE_ENGINE);
+
+ intel_engine_init_execlist(engine);
+ intel_engine_init_hangcheck(engine);
+ intel_engine_init_batch_pool(engine);
+ intel_engine_init_cmd_parser(engine);
+
+ return 0;
+
+err_hwsp:
+ cleanup_status_page(engine);
+ return err;
+}
+
static void __intel_context_unpin(struct i915_gem_context *ctx,
struct intel_engine_cs *engine)
{
@@ -617,7 +635,7 @@ struct measure_breadcrumb {
static int measure_breadcrumb_dw(struct intel_engine_cs *engine)
{
struct measure_breadcrumb *frame;
- unsigned int dw;
+ int dw = -ENOMEM;
GEM_BUG_ON(!engine->i915->gt.scratch);
@@ -625,7 +643,10 @@ static int measure_breadcrumb_dw(struct intel_engine_cs *engine)
if (!frame)
return -ENOMEM;
- i915_timeline_init(engine->i915, &frame->timeline, "measure");
+ if (i915_timeline_init(engine->i915,
+ &frame->timeline, "measure",
+ engine->status_page.vma))
+ goto out_frame;
INIT_LIST_HEAD(&frame->ring.request_list);
frame->ring.timeline = &frame->timeline;
@@ -642,8 +663,9 @@ static int measure_breadcrumb_dw(struct intel_engine_cs *engine)
dw = engine->emit_breadcrumb(&frame->rq, frame->cs) - frame->cs;
i915_timeline_fini(&frame->timeline);
- kfree(frame);
+out_frame:
+ kfree(frame);
return dw;
}
@@ -693,20 +715,14 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
if (ret)
goto err_unpin_preempt;
- ret = init_status_page(engine);
- if (ret)
- goto err_breadcrumbs;
-
ret = measure_breadcrumb_dw(engine);
if (ret < 0)
- goto err_status_page;
+ goto err_breadcrumbs;
engine->emit_breadcrumb_dw = ret;
return 0;
-err_status_page:
- cleanup_status_page(engine);
err_breadcrumbs:
intel_engine_fini_breadcrumbs(engine);
err_unpin_preempt:
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 9ae7f77293a0..e388f37743a2 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -2206,10 +2206,14 @@ logical_ring_default_irqs(struct intel_engine_cs *engine)
engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;
}
-static void
+static int
logical_ring_setup(struct intel_engine_cs *engine)
{
- intel_engine_setup_common(engine);
+ int err;
+
+ err = intel_engine_setup_common(engine);
+ if (err)
+ return err;
/* Intentionally left blank. */
engine->buffer = NULL;
@@ -2219,6 +2223,8 @@ logical_ring_setup(struct intel_engine_cs *engine)
logical_ring_default_vfuncs(engine);
logical_ring_default_irqs(engine);
+
+ return 0;
}
static int logical_ring_init(struct intel_engine_cs *engine)
@@ -2267,7 +2273,9 @@ int logical_render_ring_init(struct intel_engine_cs *engine)
{
int ret;
- logical_ring_setup(engine);
+ ret = logical_ring_setup(engine);
+ if (ret)
+ return ret;
/* Override some for render ring. */
engine->init_context = gen8_init_rcs_context;
@@ -2296,7 +2304,11 @@ int logical_render_ring_init(struct intel_engine_cs *engine)
int logical_xcs_ring_init(struct intel_engine_cs *engine)
{
- logical_ring_setup(engine);
+ int err;
+
+ err = logical_ring_setup(engine);
+ if (err)
+ return err;
return logical_ring_init(engine);
}
@@ -2629,7 +2641,7 @@ static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
goto error_deref_obj;
}
- timeline = i915_timeline_create(ctx->i915, ctx->name);
+ timeline = i915_timeline_create(ctx->i915, ctx->name, NULL);
if (IS_ERR(timeline)) {
ret = PTR_ERR(timeline);
goto error_deref_obj;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index cb6d2aa2a829..174795622eb1 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -1545,9 +1545,13 @@ static int intel_init_ring_buffer(struct intel_engine_cs *engine)
struct intel_ring *ring;
int err;
- intel_engine_setup_common(engine);
+ err = intel_engine_setup_common(engine);
+ if (err)
+ return err;
- timeline = i915_timeline_create(engine->i915, engine->name);
+ timeline = i915_timeline_create(engine->i915,
+ engine->name,
+ engine->status_page.vma);
if (IS_ERR(timeline)) {
err = PTR_ERR(timeline);
goto err;
@@ -1571,6 +1575,8 @@ static int intel_init_ring_buffer(struct intel_engine_cs *engine)
if (err)
goto err_unpin;
+ GEM_BUG_ON(ring->timeline->hwsp_ggtt != engine->status_page.vma);
+
return 0;
err_unpin:
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 32371ae67f24..2927b712b973 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -712,7 +712,9 @@ intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value)
#define I915_GEM_HWS_INDEX_ADDR (I915_GEM_HWS_INDEX * sizeof(u32))
#define I915_GEM_HWS_PREEMPT 0x32
#define I915_GEM_HWS_PREEMPT_ADDR (I915_GEM_HWS_PREEMPT * sizeof(u32))
-#define I915_GEM_HWS_SCRATCH 0x40
+#define I915_GEM_HWS_SEQNO 0x40
+#define I915_GEM_HWS_SEQNO_ADDR (I915_GEM_HWS_SEQNO * sizeof(u32))
+#define I915_GEM_HWS_SCRATCH 0x80
#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH * sizeof(u32))
#define I915_HWS_CSB_BUF0_INDEX 0x10
@@ -818,7 +820,7 @@ intel_ring_set_tail(struct intel_ring *ring, unsigned int tail)
void intel_engine_write_global_seqno(struct intel_engine_cs *engine, u32 seqno);
-void intel_engine_setup_common(struct intel_engine_cs *engine);
+int intel_engine_setup_common(struct intel_engine_cs *engine);
int intel_engine_init_common(struct intel_engine_cs *engine);
void intel_engine_cleanup_common(struct intel_engine_cs *engine);
diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
index a15713cae3b3..76b4f87fc853 100644
--- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
+++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
@@ -13,6 +13,7 @@ selftest(sanitycheck, i915_live_sanitycheck) /* keep first (igt selfcheck) */
selftest(uncore, intel_uncore_live_selftests)
selftest(workarounds, intel_workarounds_live_selftests)
selftest(requests, i915_request_live_selftests)
+selftest(timelines, i915_timeline_live_selftests)
selftest(objects, i915_gem_object_live_selftests)
selftest(dmabuf, i915_gem_dmabuf_live_selftests)
selftest(coherency, i915_gem_coherency_live_selftests)
diff --git a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
index 1b70208eeea7..4a83a1c6c406 100644
--- a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
+++ b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h
@@ -16,7 +16,7 @@ selftest(syncmap, i915_syncmap_mock_selftests)
selftest(uncore, intel_uncore_mock_selftests)
selftest(engine, intel_engine_cs_mock_selftests)
selftest(breadcrumbs, intel_breadcrumbs_mock_selftests)
-selftest(timelines, i915_gem_timeline_mock_selftests)
+selftest(timelines, i915_timeline_mock_selftests)
selftest(requests, i915_request_mock_selftests)
selftest(objects, i915_gem_object_mock_selftests)
selftest(dmabuf, i915_gem_dmabuf_mock_selftests)
diff --git a/drivers/gpu/drm/i915/selftests/i915_timeline.c b/drivers/gpu/drm/i915/selftests/i915_timeline.c
index 19f1c6a5c8fb..1585b614510d 100644
--- a/drivers/gpu/drm/i915/selftests/i915_timeline.c
+++ b/drivers/gpu/drm/i915/selftests/i915_timeline.c
@@ -7,6 +7,7 @@
#include "../i915_selftest.h"
#include "i915_random.h"
+#include "igt_flush_test.h"
#include "mock_gem_device.h"
#include "mock_timeline.h"
@@ -256,7 +257,7 @@ static int bench_sync(void *arg)
return 0;
}
-int i915_gem_timeline_mock_selftests(void)
+int i915_timeline_mock_selftests(void)
{
static const struct i915_subtest tests[] = {
SUBTEST(igt_sync),
@@ -265,3 +266,326 @@ int i915_gem_timeline_mock_selftests(void)
return i915_subtests(tests, NULL);
}
+
+static int emit_ggtt_store_dw(struct i915_request *rq, u32 addr, u32 value)
+{
+ u32 *cs;
+
+ cs = intel_ring_begin(rq, 4);
+ if (IS_ERR(cs))
+ return PTR_ERR(cs);
+
+ if (INTEL_GEN(rq->i915) >= 8) {
+ *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
+ *cs++ = addr;
+ *cs++ = 0;
+ *cs++ = value;
+ } else if (INTEL_GEN(rq->i915) >= 4) {
+ *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
+ *cs++ = 0;
+ *cs++ = addr;
+ *cs++ = value;
+ } else {
+ *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
+ *cs++ = addr;
+ *cs++ = value;
+ *cs++ = MI_NOOP;
+ }
+
+ intel_ring_advance(rq, cs);
+
+ return 0;
+}
+
+static u32 hwsp_address(const struct i915_timeline *tl)
+{
+ return i915_ggtt_offset(tl->hwsp_ggtt) + tl->hwsp_offset;
+}
+
+static struct i915_request *
+tl_write(struct i915_timeline *tl, struct intel_engine_cs *engine, u32 value)
+{
+ struct i915_request *rq;
+ int err;
+
+ lockdep_assert_held(&tl->i915->drm.struct_mutex); /* lazy rq refs */
+
+ err = i915_timeline_pin(tl);
+ if (err) {
+ rq = ERR_PTR(err);
+ goto out;
+ }
+
+ rq = i915_request_alloc(engine, engine->i915->kernel_context);
+ if (IS_ERR(rq))
+ goto out_unpin;
+
+ err = emit_ggtt_store_dw(rq, hwsp_address(tl), value);
+ i915_request_add(rq);
+ if (err)
+ rq = ERR_PTR(err);
+
+out_unpin:
+ i915_timeline_unpin(tl);
+out:
+ if (IS_ERR(rq))
+ pr_err("Failed to write to timeline!\n");
+ return rq;
+}
+
+static struct i915_timeline *
+checked_i915_timeline_create(struct drm_i915_private *i915)
+{
+ struct i915_timeline *tl;
+
+ tl = i915_timeline_create(i915, "live", NULL);
+ if (IS_ERR(tl))
+ return tl;
+
+ if (*tl->hwsp_seqno != tl->seqno) {
+ pr_err("Timeline created with incorrect breadcrumb, found %x, expected %x\n",
+ *tl->hwsp_seqno, tl->seqno);
+ i915_timeline_put(tl);
+ return ERR_PTR(-EINVAL);
+ }
+
+ return tl;
+}
+
+static int live_hwsp_engine(void *arg)
+{
+#define NUM_TIMELINES 4096
+ struct drm_i915_private *i915 = arg;
+ struct i915_timeline **timelines;
+ struct intel_engine_cs *engine;
+ enum intel_engine_id id;
+ intel_wakeref_t wakeref;
+ unsigned long count, n;
+ int err = 0;
+
+ /*
+ * Create a bunch of timelines and check we can write
+ * independently to each of their breadcrumb slots.
+ */
+
+ timelines = kvmalloc_array(NUM_TIMELINES * I915_NUM_ENGINES,
+ sizeof(*timelines),
+ GFP_KERNEL);
+ if (!timelines)
+ return -ENOMEM;
+
+ mutex_lock(&i915->drm.struct_mutex);
+ wakeref = intel_runtime_pm_get(i915);
+
+ count = 0;
+ for_each_engine(engine, i915, id) {
+ if (!intel_engine_can_store_dword(engine))
+ continue;
+
+ for (n = 0; n < NUM_TIMELINES; n++) {
+ struct i915_timeline *tl;
+ struct i915_request *rq;
+
+ tl = checked_i915_timeline_create(i915);
+ if (IS_ERR(tl)) {
+ err = PTR_ERR(tl);
+ goto out;
+ }
+
+ rq = tl_write(tl, engine, count);
+ if (IS_ERR(rq)) {
+ i915_timeline_put(tl);
+ err = PTR_ERR(rq);
+ goto out;
+ }
+
+ timelines[count++] = tl;
+ }
+ }
+
+out:
+ if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ err = -EIO;
+
+ for (n = 0; n < count; n++) {
+ struct i915_timeline *tl = timelines[n];
+
+ if (!err && *tl->hwsp_seqno != n) {
+ pr_err("Invalid seqno stored in timeline %lu, found 0x%x\n",
+ n, *tl->hwsp_seqno);
+ err = -EINVAL;
+ }
+ i915_timeline_put(tl);
+ }
+
+ intel_runtime_pm_put(i915, wakeref);
+ mutex_unlock(&i915->drm.struct_mutex);
+
+ kvfree(timelines);
+
+ return err;
+#undef NUM_TIMELINES
+}
+
+static int live_hwsp_alternate(void *arg)
+{
+#define NUM_TIMELINES 4096
+ struct drm_i915_private *i915 = arg;
+ struct i915_timeline **timelines;
+ struct intel_engine_cs *engine;
+ enum intel_engine_id id;
+ intel_wakeref_t wakeref;
+ unsigned long count, n;
+ int err = 0;
+
+ /*
+ * Create a bunch of timelines and check we can write
+ * independently to each of their breadcrumb slots with adjacent
+ * engines.
+ */
+
+ timelines = kvmalloc_array(NUM_TIMELINES * I915_NUM_ENGINES,
+ sizeof(*timelines),
+ GFP_KERNEL);
+ if (!timelines)
+ return -ENOMEM;
+
+ mutex_lock(&i915->drm.struct_mutex);
+ wakeref = intel_runtime_pm_get(i915);
+
+ count = 0;
+ for (n = 0; n < NUM_TIMELINES; n++) {
+ for_each_engine(engine, i915, id) {
+ struct i915_timeline *tl;
+ struct i915_request *rq;
+
+ if (!intel_engine_can_store_dword(engine))
+ continue;
+
+ tl = checked_i915_timeline_create(i915);
+ if (IS_ERR(tl)) {
+ err = PTR_ERR(tl);
+ goto out;
+ }
+
+ rq = tl_write(tl, engine, count);
+ if (IS_ERR(rq)) {
+ i915_timeline_put(tl);
+ err = PTR_ERR(rq);
+ goto out;
+ }
+
+ timelines[count++] = tl;
+ }
+ }
+
+out:
+ if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ err = -EIO;
+
+ for (n = 0; n < count; n++) {
+ struct i915_timeline *tl = timelines[n];
+
+ if (!err && *tl->hwsp_seqno != n) {
+ pr_err("Invalid seqno stored in timeline %lu, found 0x%x\n",
+ n, *tl->hwsp_seqno);
+ err = -EINVAL;
+ }
+ i915_timeline_put(tl);
+ }
+
+ intel_runtime_pm_put(i915, wakeref);
+ mutex_unlock(&i915->drm.struct_mutex);
+
+ kvfree(timelines);
+
+ return err;
+#undef NUM_TIMELINES
+}
+
+static int live_hwsp_recycle(void *arg)
+{
+ struct drm_i915_private *i915 = arg;
+ struct intel_engine_cs *engine;
+ enum intel_engine_id id;
+ intel_wakeref_t wakeref;
+ unsigned long count;
+ int err = 0;
+
+ /*
+ * Check seqno writes into one timeline at a time. We expect to
+ * recycle the breadcrumb slot between iterations and neither
+ * want to confuse ourselves or the GPU.
+ */
+
+ mutex_lock(&i915->drm.struct_mutex);
+ wakeref = intel_runtime_pm_get(i915);
+
+ count = 0;
+ for_each_engine(engine, i915, id) {
+ IGT_TIMEOUT(end_time);
+
+ if (!intel_engine_can_store_dword(engine))
+ continue;
+
+ do {
+ struct i915_timeline *tl;
+ struct i915_request *rq;
+
+ tl = checked_i915_timeline_create(i915);
+ if (IS_ERR(tl)) {
+ err = PTR_ERR(tl);
+ goto out;
+ }
+
+ rq = tl_write(tl, engine, count);
+ if (IS_ERR(rq)) {
+ i915_timeline_put(tl);
+ err = PTR_ERR(rq);
+ goto out;
+ }
+
+ if (i915_request_wait(rq,
+ I915_WAIT_LOCKED,
+ HZ / 5) < 0) {
+ pr_err("Wait for timeline writes timed out!\n");
+ i915_timeline_put(tl);
+ err = -EIO;
+ goto out;
+ }
+
+ if (*tl->hwsp_seqno != count) {
+ pr_err("Invalid seqno stored in timeline %lu, found 0x%x\n",
+ count, *tl->hwsp_seqno);
+ err = -EINVAL;
+ }
+
+ i915_timeline_put(tl);
+ count++;
+
+ if (err)
+ goto out;
+
+ i915_timelines_park(i915); /* Encourage recycling! */
+ } while (!__igt_timeout(end_time, NULL));
+ }
+
+out:
+ if (igt_flush_test(i915, I915_WAIT_LOCKED))
+ err = -EIO;
+ intel_runtime_pm_put(i915, wakeref);
+ mutex_unlock(&i915->drm.struct_mutex);
+
+ return err;
+}
+
+int i915_timeline_live_selftests(struct drm_i915_private *i915)
+{
+ static const struct i915_subtest tests[] = {
+ SUBTEST(live_hwsp_recycle),
+ SUBTEST(live_hwsp_engine),
+ SUBTEST(live_hwsp_alternate),
+ };
+
+ return i915_subtests(tests, i915);
+}
diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.c b/drivers/gpu/drm/i915/selftests/mock_engine.c
index 4e5b4dc6df0f..919c89fd6ee5 100644
--- a/drivers/gpu/drm/i915/selftests/mock_engine.c
+++ b/drivers/gpu/drm/i915/selftests/mock_engine.c
@@ -39,7 +39,12 @@ static struct intel_ring *mock_ring(struct intel_engine_cs *engine)
if (!ring)
return NULL;
- i915_timeline_init(engine->i915, &ring->timeline, engine->name);
+ if (i915_timeline_init(engine->i915,
+ &ring->timeline, engine->name,
+ NULL)) {
+ kfree(ring);
+ return NULL;
+ }
ring->base.size = sz;
ring->base.effective_size = sz;
@@ -208,7 +213,11 @@ struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
engine->base.emit_breadcrumb = mock_emit_breadcrumb;
engine->base.submit_request = mock_submit_request;
- i915_timeline_init(i915, &engine->base.timeline, engine->base.name);
+ if (i915_timeline_init(i915,
+ &engine->base.timeline,
+ engine->base.name,
+ NULL))
+ goto err_free;
i915_timeline_set_subclass(&engine->base.timeline, TIMELINE_ENGINE);
intel_engine_init_breadcrumbs(&engine->base);
@@ -226,6 +235,7 @@ struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
err_breadcrumbs:
intel_engine_fini_breadcrumbs(&engine->base);
i915_timeline_fini(&engine->base.timeline);
+err_free:
kfree(engine);
return NULL;
}
--
2.20.1
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [CI 4/6] drm/i915: Share per-timeline HWSP using a slab suballocator
2019-01-28 18:18 [CI 1/6] drm/i915: Introduce concept of per-timeline (context) HWSP Chris Wilson
2019-01-28 18:18 ` [CI 2/6] drm/i915: Enlarge vma->pin_count Chris Wilson
2019-01-28 18:18 ` [CI 3/6] drm/i915: Allocate a status page for each timeline Chris Wilson
@ 2019-01-28 18:18 ` Chris Wilson
2019-01-28 18:18 ` [CI 5/6] drm/i915: Track the context's seqno in its own timeline HWSP Chris Wilson
` (5 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Chris Wilson @ 2019-01-28 18:18 UTC (permalink / raw)
To: intel-gfx
If we restrict ourselves to only using a cacheline for each timeline's
HWSP (we could go smaller, but want to avoid needless polluting
cachelines on different engines between different contexts), then we can
suballocate a single 4k page into 64 different timeline HWSP. By
treating each fresh allocation as a slab of 64 entries, we can keep it
around for the next 64 allocation attempts until we need to refresh the
slab cache.
John Harrison noted the issue of fragmentation leading to the same worst
case performance of one page per timeline as before, which can be
mitigated by adopting a freelist.
v2: Keep all partially allocated HWSP on a freelist
This is still without migration, so it is possible for the system to end
up with each timeline in its own page, but we ensure that no new
allocation would needless allocate a fresh page!
v3: Throw a selftest at the allocator to try and catch invalid cacheline
reuse.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Cc: John Harrison <John.C.Harrison@Intel.com>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
---
drivers/gpu/drm/i915/i915_drv.h | 4 +
drivers/gpu/drm/i915/i915_timeline.c | 124 ++++++++++++---
drivers/gpu/drm/i915/i915_timeline.h | 1 +
drivers/gpu/drm/i915/selftests/i915_random.c | 33 +++-
drivers/gpu/drm/i915/selftests/i915_random.h | 3 +
.../gpu/drm/i915/selftests/i915_timeline.c | 143 ++++++++++++++++++
6 files changed, 280 insertions(+), 28 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 8a181b455197..6a051381f535 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -1978,6 +1978,10 @@ struct drm_i915_private {
struct i915_gt_timelines {
struct mutex mutex; /* protects list, tainted by GPU */
struct list_head list;
+
+ /* Pack multiple timelines' seqnos into the same page */
+ spinlock_t hwsp_lock;
+ struct list_head hwsp_free_list;
} timelines;
struct list_head active_rings;
diff --git a/drivers/gpu/drm/i915/i915_timeline.c b/drivers/gpu/drm/i915/i915_timeline.c
index 8d5792311a8f..add8fc33cf6e 100644
--- a/drivers/gpu/drm/i915/i915_timeline.c
+++ b/drivers/gpu/drm/i915/i915_timeline.c
@@ -9,6 +9,18 @@
#include "i915_timeline.h"
#include "i915_syncmap.h"
+struct i915_timeline_hwsp {
+ struct i915_vma *vma;
+ struct list_head free_link;
+ u64 free_bitmap;
+};
+
+static inline struct i915_timeline_hwsp *
+i915_timeline_hwsp(const struct i915_timeline *tl)
+{
+ return tl->hwsp_ggtt->private;
+}
+
static struct i915_vma *__hwsp_alloc(struct drm_i915_private *i915)
{
struct drm_i915_gem_object *obj;
@@ -27,28 +39,89 @@ static struct i915_vma *__hwsp_alloc(struct drm_i915_private *i915)
return vma;
}
-static int hwsp_alloc(struct i915_timeline *timeline)
+static struct i915_vma *
+hwsp_alloc(struct i915_timeline *timeline, unsigned int *cacheline)
{
- struct i915_vma *vma;
+ struct drm_i915_private *i915 = timeline->i915;
+ struct i915_gt_timelines *gt = &i915->gt.timelines;
+ struct i915_timeline_hwsp *hwsp;
- vma = __hwsp_alloc(timeline->i915);
- if (IS_ERR(vma))
- return PTR_ERR(vma);
+ BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);
- timeline->hwsp_ggtt = vma;
- timeline->hwsp_offset = 0;
+ spin_lock(>->hwsp_lock);
- return 0;
+ /* hwsp_free_list only contains HWSP that have available cachelines */
+ hwsp = list_first_entry_or_null(>->hwsp_free_list,
+ typeof(*hwsp), free_link);
+ if (!hwsp) {
+ struct i915_vma *vma;
+
+ spin_unlock(>->hwsp_lock);
+
+ hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
+ if (!hwsp)
+ return ERR_PTR(-ENOMEM);
+
+ vma = __hwsp_alloc(i915);
+ if (IS_ERR(vma)) {
+ kfree(hwsp);
+ return vma;
+ }
+
+ vma->private = hwsp;
+ hwsp->vma = vma;
+ hwsp->free_bitmap = ~0ull;
+
+ spin_lock(>->hwsp_lock);
+ list_add(&hwsp->free_link, >->hwsp_free_list);
+ }
+
+ GEM_BUG_ON(!hwsp->free_bitmap);
+ *cacheline = __ffs64(hwsp->free_bitmap);
+ hwsp->free_bitmap &= ~BIT_ULL(*cacheline);
+ if (!hwsp->free_bitmap)
+ list_del(&hwsp->free_link);
+
+ spin_unlock(>->hwsp_lock);
+
+ GEM_BUG_ON(hwsp->vma->private != hwsp);
+ return hwsp->vma;
+}
+
+static void hwsp_free(struct i915_timeline *timeline)
+{
+ struct i915_gt_timelines *gt = &timeline->i915->gt.timelines;
+ struct i915_timeline_hwsp *hwsp;
+
+ hwsp = i915_timeline_hwsp(timeline);
+ if (!hwsp) /* leave global HWSP alone! */
+ return;
+
+ spin_lock(>->hwsp_lock);
+
+ /* As a cacheline becomes available, publish the HWSP on the freelist */
+ if (!hwsp->free_bitmap)
+ list_add_tail(&hwsp->free_link, >->hwsp_free_list);
+
+ hwsp->free_bitmap |= BIT_ULL(timeline->hwsp_offset / CACHELINE_BYTES);
+
+ /* And if no one is left using it, give the page back to the system */
+ if (hwsp->free_bitmap == ~0ull) {
+ i915_vma_put(hwsp->vma);
+ list_del(&hwsp->free_link);
+ kfree(hwsp);
+ }
+
+ spin_unlock(>->hwsp_lock);
}
int i915_timeline_init(struct drm_i915_private *i915,
struct i915_timeline *timeline,
const char *name,
- struct i915_vma *global_hwsp)
+ struct i915_vma *hwsp)
{
struct i915_gt_timelines *gt = &i915->gt.timelines;
void *vaddr;
- int err;
/*
* Ideally we want a set of engines on a single leaf as we expect
@@ -64,18 +137,22 @@ int i915_timeline_init(struct drm_i915_private *i915,
timeline->name = name;
timeline->pin_count = 0;
- if (global_hwsp) {
- timeline->hwsp_ggtt = i915_vma_get(global_hwsp);
- timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR;
- } else {
- err = hwsp_alloc(timeline);
- if (err)
- return err;
+ timeline->hwsp_offset = I915_GEM_HWS_SEQNO_ADDR;
+ if (!hwsp) {
+ unsigned int cacheline;
+
+ hwsp = hwsp_alloc(timeline, &cacheline);
+ if (IS_ERR(hwsp))
+ return PTR_ERR(hwsp);
+
+ timeline->hwsp_offset = cacheline * CACHELINE_BYTES;
}
+ timeline->hwsp_ggtt = i915_vma_get(hwsp);
- vaddr = i915_gem_object_pin_map(timeline->hwsp_ggtt->obj, I915_MAP_WB);
+ vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
if (IS_ERR(vaddr)) {
- i915_vma_put(timeline->hwsp_ggtt);
+ hwsp_free(timeline);
+ i915_vma_put(hwsp);
return PTR_ERR(vaddr);
}
@@ -105,6 +182,9 @@ void i915_timelines_init(struct drm_i915_private *i915)
mutex_init(>->mutex);
INIT_LIST_HEAD(>->list);
+ spin_lock_init(>->hwsp_lock);
+ INIT_LIST_HEAD(>->hwsp_free_list);
+
/* via i915_gem_wait_for_idle() */
i915_gem_shrinker_taints_mutex(i915, >->mutex);
}
@@ -144,12 +224,13 @@ void i915_timeline_fini(struct i915_timeline *timeline)
GEM_BUG_ON(timeline->pin_count);
GEM_BUG_ON(!list_empty(&timeline->requests));
- i915_syncmap_free(&timeline->sync);
-
mutex_lock(>->mutex);
list_del(&timeline->link);
mutex_unlock(>->mutex);
+ i915_syncmap_free(&timeline->sync);
+ hwsp_free(timeline);
+
i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
i915_vma_put(timeline->hwsp_ggtt);
}
@@ -226,6 +307,7 @@ void i915_timelines_fini(struct drm_i915_private *i915)
struct i915_gt_timelines *gt = &i915->gt.timelines;
GEM_BUG_ON(!list_empty(>->list));
+ GEM_BUG_ON(!list_empty(>->hwsp_free_list));
mutex_destroy(>->mutex);
}
diff --git a/drivers/gpu/drm/i915/i915_timeline.h b/drivers/gpu/drm/i915/i915_timeline.h
index 0c3739d53d79..ab736e2e5707 100644
--- a/drivers/gpu/drm/i915/i915_timeline.h
+++ b/drivers/gpu/drm/i915/i915_timeline.h
@@ -33,6 +33,7 @@
#include "i915_utils.h"
struct i915_vma;
+struct i915_timeline_hwsp;
struct i915_timeline {
u64 fence_context;
diff --git a/drivers/gpu/drm/i915/selftests/i915_random.c b/drivers/gpu/drm/i915/selftests/i915_random.c
index 1f415ce47018..716a3f19f030 100644
--- a/drivers/gpu/drm/i915/selftests/i915_random.c
+++ b/drivers/gpu/drm/i915/selftests/i915_random.c
@@ -41,18 +41,37 @@ u64 i915_prandom_u64_state(struct rnd_state *rnd)
return x;
}
-void i915_random_reorder(unsigned int *order, unsigned int count,
- struct rnd_state *state)
+void i915_prandom_shuffle(void *arr, size_t elsz, size_t count,
+ struct rnd_state *state)
{
- unsigned int i, j;
+ char stack[128];
+
+ if (WARN_ON(elsz > sizeof(stack) || count > U32_MAX))
+ return;
+
+ if (!elsz || !count)
+ return;
+
+ /* Fisher-Yates shuffle courtesy of Knuth */
+ while (--count) {
+ size_t swp;
+
+ swp = i915_prandom_u32_max_state(count + 1, state);
+ if (swp == count)
+ continue;
- for (i = 0; i < count; i++) {
- BUILD_BUG_ON(sizeof(unsigned int) > sizeof(u32));
- j = i915_prandom_u32_max_state(count, state);
- swap(order[i], order[j]);
+ memcpy(stack, arr + count * elsz, elsz);
+ memcpy(arr + count * elsz, arr + swp * elsz, elsz);
+ memcpy(arr + swp * elsz, stack, elsz);
}
}
+void i915_random_reorder(unsigned int *order, unsigned int count,
+ struct rnd_state *state)
+{
+ i915_prandom_shuffle(order, sizeof(*order), count, state);
+}
+
unsigned int *i915_random_order(unsigned int count, struct rnd_state *state)
{
unsigned int *order, i;
diff --git a/drivers/gpu/drm/i915/selftests/i915_random.h b/drivers/gpu/drm/i915/selftests/i915_random.h
index 7dffedc501ca..8e1ff9c105b6 100644
--- a/drivers/gpu/drm/i915/selftests/i915_random.h
+++ b/drivers/gpu/drm/i915/selftests/i915_random.h
@@ -54,4 +54,7 @@ void i915_random_reorder(unsigned int *order,
unsigned int count,
struct rnd_state *state);
+void i915_prandom_shuffle(void *arr, size_t elsz, size_t count,
+ struct rnd_state *state);
+
#endif /* !__I915_SELFTESTS_RANDOM_H__ */
diff --git a/drivers/gpu/drm/i915/selftests/i915_timeline.c b/drivers/gpu/drm/i915/selftests/i915_timeline.c
index 1585b614510d..c34340f074cf 100644
--- a/drivers/gpu/drm/i915/selftests/i915_timeline.c
+++ b/drivers/gpu/drm/i915/selftests/i915_timeline.c
@@ -4,6 +4,8 @@
* Copyright © 2017-2018 Intel Corporation
*/
+#include <linux/prime_numbers.h>
+
#include "../i915_selftest.h"
#include "i915_random.h"
@@ -11,6 +13,146 @@
#include "mock_gem_device.h"
#include "mock_timeline.h"
+static struct page *hwsp_page(struct i915_timeline *tl)
+{
+ struct drm_i915_gem_object *obj = tl->hwsp_ggtt->obj;
+
+ GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
+ return sg_page(obj->mm.pages->sgl);
+}
+
+static unsigned long hwsp_cacheline(struct i915_timeline *tl)
+{
+ unsigned long address = (unsigned long)page_address(hwsp_page(tl));
+
+ return (address + tl->hwsp_offset) / CACHELINE_BYTES;
+}
+
+#define CACHELINES_PER_PAGE (PAGE_SIZE / CACHELINE_BYTES)
+
+struct mock_hwsp_freelist {
+ struct drm_i915_private *i915;
+ struct radix_tree_root cachelines;
+ struct i915_timeline **history;
+ unsigned long count, max;
+ struct rnd_state prng;
+};
+
+enum {
+ SHUFFLE = BIT(0),
+};
+
+static void __mock_hwsp_record(struct mock_hwsp_freelist *state,
+ unsigned int idx,
+ struct i915_timeline *tl)
+{
+ tl = xchg(&state->history[idx], tl);
+ if (tl) {
+ radix_tree_delete(&state->cachelines, hwsp_cacheline(tl));
+ i915_timeline_put(tl);
+ }
+}
+
+static int __mock_hwsp_timeline(struct mock_hwsp_freelist *state,
+ unsigned int count,
+ unsigned int flags)
+{
+ struct i915_timeline *tl;
+ unsigned int idx;
+
+ while (count--) {
+ unsigned long cacheline;
+ int err;
+
+ tl = i915_timeline_create(state->i915, "mock", NULL);
+ if (IS_ERR(tl))
+ return PTR_ERR(tl);
+
+ cacheline = hwsp_cacheline(tl);
+ err = radix_tree_insert(&state->cachelines, cacheline, tl);
+ if (err) {
+ if (err == -EEXIST) {
+ pr_err("HWSP cacheline %lu already used; duplicate allocation!\n",
+ cacheline);
+ }
+ i915_timeline_put(tl);
+ return err;
+ }
+
+ idx = state->count++ % state->max;
+ __mock_hwsp_record(state, idx, tl);
+ }
+
+ if (flags & SHUFFLE)
+ i915_prandom_shuffle(state->history,
+ sizeof(*state->history),
+ min(state->count, state->max),
+ &state->prng);
+
+ count = i915_prandom_u32_max_state(min(state->count, state->max),
+ &state->prng);
+ while (count--) {
+ idx = --state->count % state->max;
+ __mock_hwsp_record(state, idx, NULL);
+ }
+
+ return 0;
+}
+
+static int mock_hwsp_freelist(void *arg)
+{
+ struct mock_hwsp_freelist state;
+ const struct {
+ const char *name;
+ unsigned int flags;
+ } phases[] = {
+ { "linear", 0 },
+ { "shuffled", SHUFFLE },
+ { },
+ }, *p;
+ unsigned int na;
+ int err = 0;
+
+ INIT_RADIX_TREE(&state.cachelines, GFP_KERNEL);
+ state.prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed);
+
+ state.i915 = mock_gem_device();
+ if (!state.i915)
+ return -ENOMEM;
+
+ /*
+ * Create a bunch of timelines and check that their HWSP do not overlap.
+ * Free some, and try again.
+ */
+
+ state.max = PAGE_SIZE / sizeof(*state.history);
+ state.count = 0;
+ state.history = kcalloc(state.max, sizeof(*state.history), GFP_KERNEL);
+ if (!state.history) {
+ err = -ENOMEM;
+ goto err_put;
+ }
+
+ mutex_lock(&state.i915->drm.struct_mutex);
+ for (p = phases; p->name; p++) {
+ pr_debug("%s(%s)\n", __func__, p->name);
+ for_each_prime_number_from(na, 1, 2 * CACHELINES_PER_PAGE) {
+ err = __mock_hwsp_timeline(&state, na, p->flags);
+ if (err)
+ goto out;
+ }
+ }
+
+out:
+ for (na = 0; na < state.max; na++)
+ __mock_hwsp_record(&state, na, NULL);
+ mutex_unlock(&state.i915->drm.struct_mutex);
+ kfree(state.history);
+err_put:
+ drm_dev_put(&state.i915->drm);
+ return err;
+}
+
struct __igt_sync {
const char *name;
u32 seqno;
@@ -260,6 +402,7 @@ static int bench_sync(void *arg)
int i915_timeline_mock_selftests(void)
{
static const struct i915_subtest tests[] = {
+ SUBTEST(mock_hwsp_freelist),
SUBTEST(igt_sync),
SUBTEST(bench_sync),
};
--
2.20.1
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [CI 5/6] drm/i915: Track the context's seqno in its own timeline HWSP
2019-01-28 18:18 [CI 1/6] drm/i915: Introduce concept of per-timeline (context) HWSP Chris Wilson
` (2 preceding siblings ...)
2019-01-28 18:18 ` [CI 4/6] drm/i915: Share per-timeline HWSP using a slab suballocator Chris Wilson
@ 2019-01-28 18:18 ` Chris Wilson
2019-01-28 18:18 ` [CI 6/6] drm/i915: Track active timelines Chris Wilson
` (4 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Chris Wilson @ 2019-01-28 18:18 UTC (permalink / raw)
To: intel-gfx
Now that we have allocated ourselves a cacheline to store a breadcrumb,
we can emit a write from the GPU into the timeline's HWSP of the
per-context seqno as we complete each request. This drops the mirroring
of the per-engine HWSP and allows each context to operate independently.
We do not need to unwind the per-context timeline, and so requests are
always consistent with the timeline breadcrumb, greatly simplifying the
completion checks as we no longer need to be concerned about the
global_seqno changing mid check.
One complication though is that we have to be wary that the request may
outlive the HWSP and so avoid touching the potentially danging pointer
after we have retired the fence. We also have to guard our access of the
HWSP with RCU, the release of the obj->mm.pages should already be RCU-safe.
At this point, we are emitting both per-context and global seqno and
still using the single per-engine execution timeline for resolving
interrupts.
v2: s/fake_complete/mark_complete/
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
---
drivers/gpu/drm/i915/i915_gem.c | 2 +-
drivers/gpu/drm/i915/i915_request.c | 3 +-
drivers/gpu/drm/i915/i915_request.h | 30 +++----
drivers/gpu/drm/i915/i915_reset.c | 1 +
drivers/gpu/drm/i915/i915_timeline.c | 4 +
drivers/gpu/drm/i915/intel_engine_cs.c | 15 +++-
drivers/gpu/drm/i915/intel_lrc.c | 31 ++++---
drivers/gpu/drm/i915/intel_ringbuffer.c | 87 +++++++++++++++----
.../gpu/drm/i915/selftests/i915_timeline.c | 7 +-
drivers/gpu/drm/i915/selftests/mock_engine.c | 19 +++-
10 files changed, 139 insertions(+), 60 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index d68f3fdd8a8e..1bd724d663d9 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -2892,7 +2892,7 @@ i915_gem_find_active_request(struct intel_engine_cs *engine)
*/
spin_lock_irqsave(&engine->timeline.lock, flags);
list_for_each_entry(request, &engine->timeline.requests, link) {
- if (__i915_request_completed(request, request->global_seqno))
+ if (i915_request_completed(request))
continue;
active = request;
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index a076fd0b7ba6..4d58770e6a8c 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -199,6 +199,7 @@ static void __retire_engine_request(struct intel_engine_cs *engine,
spin_unlock(&engine->timeline.lock);
spin_lock(&rq->lock);
+ i915_request_mark_complete(rq);
if (!i915_request_signaled(rq))
dma_fence_signal_locked(&rq->fence);
if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &rq->fence.flags))
@@ -621,7 +622,7 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
rq->ring = ce->ring;
rq->timeline = ce->ring->timeline;
GEM_BUG_ON(rq->timeline == &engine->timeline);
- rq->hwsp_seqno = &engine->status_page.addr[I915_GEM_HWS_INDEX];
+ rq->hwsp_seqno = rq->timeline->hwsp_seqno;
spin_lock_init(&rq->lock);
dma_fence_init(&rq->fence,
diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h
index ade010fe6e26..96c586d6ff4d 100644
--- a/drivers/gpu/drm/i915/i915_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -289,6 +289,7 @@ long i915_request_wait(struct i915_request *rq,
static inline bool i915_request_signaled(const struct i915_request *rq)
{
+ /* The request may live longer than its HWSP, so check flags first! */
return test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags);
}
@@ -340,32 +341,23 @@ static inline u32 hwsp_seqno(const struct i915_request *rq)
*/
static inline bool i915_request_started(const struct i915_request *rq)
{
- u32 seqno;
-
- seqno = i915_request_global_seqno(rq);
- if (!seqno) /* not yet submitted to HW */
- return false;
+ if (i915_request_signaled(rq))
+ return true;
- return i915_seqno_passed(hwsp_seqno(rq), seqno - 1);
-}
-
-static inline bool
-__i915_request_completed(const struct i915_request *rq, u32 seqno)
-{
- GEM_BUG_ON(!seqno);
- return i915_seqno_passed(hwsp_seqno(rq), seqno) &&
- seqno == i915_request_global_seqno(rq);
+ return i915_seqno_passed(hwsp_seqno(rq), rq->fence.seqno - 1);
}
static inline bool i915_request_completed(const struct i915_request *rq)
{
- u32 seqno;
+ if (i915_request_signaled(rq))
+ return true;
- seqno = i915_request_global_seqno(rq);
- if (!seqno)
- return false;
+ return i915_seqno_passed(hwsp_seqno(rq), rq->fence.seqno);
+}
- return __i915_request_completed(rq, seqno);
+static inline void i915_request_mark_complete(struct i915_request *rq)
+{
+ rq->hwsp_seqno = (u32 *)&rq->fence.seqno; /* decouple from HWSP */
}
void i915_retire_requests(struct drm_i915_private *i915);
diff --git a/drivers/gpu/drm/i915/i915_reset.c b/drivers/gpu/drm/i915/i915_reset.c
index d2dca85a543d..bd82f9b1043f 100644
--- a/drivers/gpu/drm/i915/i915_reset.c
+++ b/drivers/gpu/drm/i915/i915_reset.c
@@ -760,6 +760,7 @@ static void nop_submit_request(struct i915_request *request)
spin_lock_irqsave(&request->engine->timeline.lock, flags);
__i915_request_submit(request);
+ i915_request_mark_complete(request);
intel_engine_write_global_seqno(request->engine, request->global_seqno);
spin_unlock_irqrestore(&request->engine->timeline.lock, flags);
}
diff --git a/drivers/gpu/drm/i915/i915_timeline.c b/drivers/gpu/drm/i915/i915_timeline.c
index add8fc33cf6e..e4c11414a824 100644
--- a/drivers/gpu/drm/i915/i915_timeline.c
+++ b/drivers/gpu/drm/i915/i915_timeline.c
@@ -270,6 +270,10 @@ int i915_timeline_pin(struct i915_timeline *tl)
if (err)
goto unpin;
+ tl->hwsp_offset =
+ i915_ggtt_offset(tl->hwsp_ggtt) +
+ offset_in_page(tl->hwsp_offset);
+
return 0;
unpin:
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index 515e87846afd..ead9c4371fe1 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -660,10 +660,16 @@ static int measure_breadcrumb_dw(struct intel_engine_cs *engine)
frame->rq.ring = &frame->ring;
frame->rq.timeline = &frame->timeline;
+ dw = i915_timeline_pin(&frame->timeline);
+ if (dw < 0)
+ goto out_timeline;
+
dw = engine->emit_breadcrumb(&frame->rq, frame->cs) - frame->cs;
- i915_timeline_fini(&frame->timeline);
+ i915_timeline_unpin(&frame->timeline);
+out_timeline:
+ i915_timeline_fini(&frame->timeline);
out_frame:
kfree(frame);
return dw;
@@ -1426,9 +1432,10 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine,
char hdr[80];
snprintf(hdr, sizeof(hdr),
- "\t\tELSP[%d] count=%d, ring->start=%08x, rq: ",
+ "\t\tELSP[%d] count=%d, ring:{start:%08x, hwsp:%08x}, rq: ",
idx, count,
- i915_ggtt_offset(rq->ring->vma));
+ i915_ggtt_offset(rq->ring->vma),
+ rq->timeline->hwsp_offset);
print_request(m, rq, hdr);
} else {
drm_printf(m, "\t\tELSP[%d] idle\n", idx);
@@ -1538,6 +1545,8 @@ void intel_engine_dump(struct intel_engine_cs *engine,
rq->ring->emit);
drm_printf(m, "\t\tring->space: 0x%08x\n",
rq->ring->space);
+ drm_printf(m, "\t\tring->hwsp: 0x%08x\n",
+ rq->timeline->hwsp_offset);
print_request_ring(m, rq);
}
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index e388f37743a2..fdbb3fe8eac9 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -832,10 +832,10 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
list_for_each_entry(rq, &engine->timeline.requests, link) {
GEM_BUG_ON(!rq->global_seqno);
- if (i915_request_signaled(rq))
- continue;
+ if (!i915_request_signaled(rq))
+ dma_fence_set_error(&rq->fence, -EIO);
- dma_fence_set_error(&rq->fence, -EIO);
+ i915_request_mark_complete(rq);
}
/* Flush the queued requests to the timeline list (for retiring). */
@@ -845,9 +845,9 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
priolist_for_each_request_consume(rq, rn, p, i) {
list_del_init(&rq->sched.link);
-
- dma_fence_set_error(&rq->fence, -EIO);
__i915_request_submit(rq);
+ dma_fence_set_error(&rq->fence, -EIO);
+ i915_request_mark_complete(rq);
}
rb_erase_cached(&p->node, &execlists->queue);
@@ -2044,10 +2044,17 @@ static u32 *gen8_emit_breadcrumb(struct i915_request *request, u32 *cs)
/* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5));
- cs = gen8_emit_ggtt_write(cs, request->global_seqno,
+ cs = gen8_emit_ggtt_write(cs,
+ request->fence.seqno,
+ request->timeline->hwsp_offset);
+
+ cs = gen8_emit_ggtt_write(cs,
+ request->global_seqno,
intel_hws_seqno_address(request->engine));
+
*cs++ = MI_USER_INTERRUPT;
*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
+
request->tail = intel_ring_offset(request, cs);
assert_ring_tail_valid(request->ring, request->tail);
@@ -2056,18 +2063,20 @@ static u32 *gen8_emit_breadcrumb(struct i915_request *request, u32 *cs)
static u32 *gen8_emit_breadcrumb_rcs(struct i915_request *request, u32 *cs)
{
- /* We're using qword write, seqno should be aligned to 8 bytes. */
- BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1);
-
cs = gen8_emit_ggtt_write_rcs(cs,
- request->global_seqno,
- intel_hws_seqno_address(request->engine),
+ request->fence.seqno,
+ request->timeline->hwsp_offset,
PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
PIPE_CONTROL_DEPTH_CACHE_FLUSH |
PIPE_CONTROL_DC_FLUSH_ENABLE |
PIPE_CONTROL_FLUSH_ENABLE |
PIPE_CONTROL_CS_STALL);
+ cs = gen8_emit_ggtt_write_rcs(cs,
+ request->global_seqno,
+ intel_hws_seqno_address(request->engine),
+ PIPE_CONTROL_CS_STALL);
+
*cs++ = MI_USER_INTERRUPT;
*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 174795622eb1..ee3719324e2d 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -326,6 +326,11 @@ static u32 *gen6_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
PIPE_CONTROL_DC_FLUSH_ENABLE |
PIPE_CONTROL_QW_WRITE |
PIPE_CONTROL_CS_STALL);
+ *cs++ = rq->timeline->hwsp_offset | PIPE_CONTROL_GLOBAL_GTT;
+ *cs++ = rq->fence.seqno;
+
+ *cs++ = GFX_OP_PIPE_CONTROL(4);
+ *cs++ = PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
*cs++ = intel_hws_seqno_address(rq->engine) | PIPE_CONTROL_GLOBAL_GTT;
*cs++ = rq->global_seqno;
@@ -427,6 +432,13 @@ static u32 *gen7_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
PIPE_CONTROL_QW_WRITE |
PIPE_CONTROL_GLOBAL_GTT_IVB |
PIPE_CONTROL_CS_STALL);
+ *cs++ = rq->timeline->hwsp_offset;
+ *cs++ = rq->fence.seqno;
+
+ *cs++ = GFX_OP_PIPE_CONTROL(4);
+ *cs++ = (PIPE_CONTROL_QW_WRITE |
+ PIPE_CONTROL_GLOBAL_GTT_IVB |
+ PIPE_CONTROL_CS_STALL);
*cs++ = intel_hws_seqno_address(rq->engine);
*cs++ = rq->global_seqno;
@@ -441,10 +453,19 @@ static u32 *gen7_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
static u32 *gen6_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
{
- *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW;
- *cs++ = intel_hws_seqno_address(rq->engine) | MI_FLUSH_DW_USE_GTT;
+ GEM_BUG_ON(rq->timeline->hwsp_ggtt != rq->engine->status_page.vma);
+ GEM_BUG_ON(offset_in_page(rq->timeline->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
+
+ *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;
+ *cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT;
+ *cs++ = rq->fence.seqno;
+
+ *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;
+ *cs++ = I915_GEM_HWS_INDEX_ADDR | MI_FLUSH_DW_USE_GTT;
*cs++ = rq->global_seqno;
+
*cs++ = MI_USER_INTERRUPT;
+ *cs++ = MI_NOOP;
rq->tail = intel_ring_offset(rq, cs);
assert_ring_tail_valid(rq->ring, rq->tail);
@@ -457,14 +478,21 @@ static u32 *gen7_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
{
int i;
- *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW;
- *cs++ = intel_hws_seqno_address(rq->engine) | MI_FLUSH_DW_USE_GTT;
+ GEM_BUG_ON(rq->timeline->hwsp_ggtt != rq->engine->status_page.vma);
+ GEM_BUG_ON(offset_in_page(rq->timeline->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
+
+ *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;
+ *cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT;
+ *cs++ = rq->fence.seqno;
+
+ *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;
+ *cs++ = I915_GEM_HWS_INDEX_ADDR | MI_FLUSH_DW_USE_GTT;
*cs++ = rq->global_seqno;
for (i = 0; i < GEN7_XCS_WA; i++) {
*cs++ = MI_STORE_DWORD_INDEX;
- *cs++ = I915_GEM_HWS_INDEX_ADDR;
- *cs++ = rq->global_seqno;
+ *cs++ = I915_GEM_HWS_SEQNO_ADDR;
+ *cs++ = rq->fence.seqno;
}
*cs++ = MI_FLUSH_DW;
@@ -472,7 +500,6 @@ static u32 *gen7_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
*cs++ = 0;
*cs++ = MI_USER_INTERRUPT;
- *cs++ = MI_NOOP;
rq->tail = intel_ring_offset(rq, cs);
assert_ring_tail_valid(rq->ring, rq->tail);
@@ -738,7 +765,7 @@ static void reset_ring(struct intel_engine_cs *engine, bool stalled)
rq = NULL;
spin_lock_irqsave(&tl->lock, flags);
list_for_each_entry(pos, &tl->requests, link) {
- if (!__i915_request_completed(pos, pos->global_seqno)) {
+ if (!i915_request_completed(pos)) {
rq = pos;
break;
}
@@ -880,10 +907,10 @@ static void cancel_requests(struct intel_engine_cs *engine)
list_for_each_entry(request, &engine->timeline.requests, link) {
GEM_BUG_ON(!request->global_seqno);
- if (i915_request_signaled(request))
- continue;
+ if (!i915_request_signaled(request))
+ dma_fence_set_error(&request->fence, -EIO);
- dma_fence_set_error(&request->fence, -EIO);
+ i915_request_mark_complete(request);
}
intel_write_status_page(engine,
@@ -907,14 +934,20 @@ static void i9xx_submit_request(struct i915_request *request)
static u32 *i9xx_emit_breadcrumb(struct i915_request *rq, u32 *cs)
{
+ GEM_BUG_ON(rq->timeline->hwsp_ggtt != rq->engine->status_page.vma);
+ GEM_BUG_ON(offset_in_page(rq->timeline->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
+
*cs++ = MI_FLUSH;
+ *cs++ = MI_STORE_DWORD_INDEX;
+ *cs++ = I915_GEM_HWS_SEQNO_ADDR;
+ *cs++ = rq->fence.seqno;
+
*cs++ = MI_STORE_DWORD_INDEX;
*cs++ = I915_GEM_HWS_INDEX_ADDR;
*cs++ = rq->global_seqno;
*cs++ = MI_USER_INTERRUPT;
- *cs++ = MI_NOOP;
rq->tail = intel_ring_offset(rq, cs);
assert_ring_tail_valid(rq->ring, rq->tail);
@@ -927,8 +960,15 @@ static u32 *gen5_emit_breadcrumb(struct i915_request *rq, u32 *cs)
{
int i;
+ GEM_BUG_ON(rq->timeline->hwsp_ggtt != rq->engine->status_page.vma);
+ GEM_BUG_ON(offset_in_page(rq->timeline->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
+
*cs++ = MI_FLUSH;
+ *cs++ = MI_STORE_DWORD_INDEX;
+ *cs++ = I915_GEM_HWS_SEQNO_ADDR;
+ *cs++ = rq->fence.seqno;
+
BUILD_BUG_ON(GEN5_WA_STORES < 1);
for (i = 0; i < GEN5_WA_STORES; i++) {
*cs++ = MI_STORE_DWORD_INDEX;
@@ -937,6 +977,7 @@ static u32 *gen5_emit_breadcrumb(struct i915_request *rq, u32 *cs)
}
*cs++ = MI_USER_INTERRUPT;
+ *cs++ = MI_NOOP;
rq->tail = intel_ring_offset(rq, cs);
assert_ring_tail_valid(rq->ring, rq->tail);
@@ -1169,6 +1210,10 @@ int intel_ring_pin(struct intel_ring *ring)
GEM_BUG_ON(ring->vaddr);
+ ret = i915_timeline_pin(ring->timeline);
+ if (ret)
+ return ret;
+
flags = PIN_GLOBAL;
/* Ring wraparound at offset 0 sometimes hangs. No idea why. */
@@ -1185,28 +1230,32 @@ int intel_ring_pin(struct intel_ring *ring)
else
ret = i915_gem_object_set_to_cpu_domain(vma->obj, true);
if (unlikely(ret))
- return ret;
+ goto unpin_timeline;
}
ret = i915_vma_pin(vma, 0, 0, flags);
if (unlikely(ret))
- return ret;
+ goto unpin_timeline;
if (i915_vma_is_map_and_fenceable(vma))
addr = (void __force *)i915_vma_pin_iomap(vma);
else
addr = i915_gem_object_pin_map(vma->obj, map);
- if (IS_ERR(addr))
- goto err;
+ if (IS_ERR(addr)) {
+ ret = PTR_ERR(addr);
+ goto unpin_ring;
+ }
vma->obj->pin_global++;
ring->vaddr = addr;
return 0;
-err:
+unpin_ring:
i915_vma_unpin(vma);
- return PTR_ERR(addr);
+unpin_timeline:
+ i915_timeline_unpin(ring->timeline);
+ return ret;
}
void intel_ring_reset(struct intel_ring *ring, u32 tail)
@@ -1235,6 +1284,8 @@ void intel_ring_unpin(struct intel_ring *ring)
ring->vma->obj->pin_global--;
i915_vma_unpin(ring->vma);
+
+ i915_timeline_unpin(ring->timeline);
}
static struct i915_vma *
diff --git a/drivers/gpu/drm/i915/selftests/i915_timeline.c b/drivers/gpu/drm/i915/selftests/i915_timeline.c
index c34340f074cf..12ea69b1a1e5 100644
--- a/drivers/gpu/drm/i915/selftests/i915_timeline.c
+++ b/drivers/gpu/drm/i915/selftests/i915_timeline.c
@@ -440,11 +440,6 @@ static int emit_ggtt_store_dw(struct i915_request *rq, u32 addr, u32 value)
return 0;
}
-static u32 hwsp_address(const struct i915_timeline *tl)
-{
- return i915_ggtt_offset(tl->hwsp_ggtt) + tl->hwsp_offset;
-}
-
static struct i915_request *
tl_write(struct i915_timeline *tl, struct intel_engine_cs *engine, u32 value)
{
@@ -463,7 +458,7 @@ tl_write(struct i915_timeline *tl, struct intel_engine_cs *engine, u32 value)
if (IS_ERR(rq))
goto out_unpin;
- err = emit_ggtt_store_dw(rq, hwsp_address(tl), value);
+ err = emit_ggtt_store_dw(rq, tl->hwsp_offset, value);
i915_request_add(rq);
if (err)
rq = ERR_PTR(err);
diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.c b/drivers/gpu/drm/i915/selftests/mock_engine.c
index 919c89fd6ee5..95e890d7f58b 100644
--- a/drivers/gpu/drm/i915/selftests/mock_engine.c
+++ b/drivers/gpu/drm/i915/selftests/mock_engine.c
@@ -30,6 +30,17 @@ struct mock_ring {
struct i915_timeline timeline;
};
+static void mock_timeline_pin(struct i915_timeline *tl)
+{
+ tl->pin_count++;
+}
+
+static void mock_timeline_unpin(struct i915_timeline *tl)
+{
+ GEM_BUG_ON(!tl->pin_count);
+ tl->pin_count--;
+}
+
static struct intel_ring *mock_ring(struct intel_engine_cs *engine)
{
const unsigned long sz = PAGE_SIZE / 2;
@@ -76,6 +87,8 @@ static void advance(struct mock_request *request)
{
list_del_init(&request->link);
mock_seqno_advance(request->base.engine, request->base.global_seqno);
+ i915_request_mark_complete(&request->base);
+ GEM_BUG_ON(!i915_request_completed(&request->base));
}
static void hw_delay_complete(struct timer_list *t)
@@ -108,6 +121,7 @@ static void hw_delay_complete(struct timer_list *t)
static void mock_context_unpin(struct intel_context *ce)
{
+ mock_timeline_unpin(ce->ring->timeline);
i915_gem_context_put(ce->gem_context);
}
@@ -129,6 +143,7 @@ mock_context_pin(struct intel_engine_cs *engine,
struct i915_gem_context *ctx)
{
struct intel_context *ce = to_intel_context(ctx, engine);
+ int err = -ENOMEM;
if (ce->pin_count++)
return ce;
@@ -139,13 +154,15 @@ mock_context_pin(struct intel_engine_cs *engine,
goto err;
}
+ mock_timeline_pin(ce->ring->timeline);
+
ce->ops = &mock_context_ops;
i915_gem_context_get(ctx);
return ce;
err:
ce->pin_count = 0;
- return ERR_PTR(-ENOMEM);
+ return ERR_PTR(err);
}
static int mock_request_alloc(struct i915_request *request)
--
2.20.1
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 10+ messages in thread
* [CI 6/6] drm/i915: Track active timelines
2019-01-28 18:18 [CI 1/6] drm/i915: Introduce concept of per-timeline (context) HWSP Chris Wilson
` (3 preceding siblings ...)
2019-01-28 18:18 ` [CI 5/6] drm/i915: Track the context's seqno in its own timeline HWSP Chris Wilson
@ 2019-01-28 18:18 ` Chris Wilson
2019-01-28 18:55 ` ✗ Fi.CI.CHECKPATCH: warning for series starting with [CI,1/6] drm/i915: Introduce concept of per-timeline (context) HWSP Patchwork
` (3 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Chris Wilson @ 2019-01-28 18:18 UTC (permalink / raw)
To: intel-gfx
Now that we pin timelines around use, we have a clearly defined lifetime
and convenient points at which we can track only the active timelines.
This allows us to reduce the list iteration to only consider those
active timelines and not all.
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
Reviewed-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
---
drivers/gpu/drm/i915/i915_drv.h | 2 +-
drivers/gpu/drm/i915/i915_gem.c | 4 +--
drivers/gpu/drm/i915/i915_reset.c | 2 +-
drivers/gpu/drm/i915/i915_timeline.c | 39 ++++++++++++++++++----------
4 files changed, 29 insertions(+), 18 deletions(-)
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 6a051381f535..d072f3369ee1 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -1977,7 +1977,7 @@ struct drm_i915_private {
struct i915_gt_timelines {
struct mutex mutex; /* protects list, tainted by GPU */
- struct list_head list;
+ struct list_head active_list;
/* Pack multiple timelines' seqnos into the same page */
spinlock_t hwsp_lock;
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 1bd724d663d9..05627000b77d 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -3248,7 +3248,7 @@ wait_for_timelines(struct drm_i915_private *i915,
return timeout;
mutex_lock(>->mutex);
- list_for_each_entry(tl, >->list, link) {
+ list_for_each_entry(tl, >->active_list, link) {
struct i915_request *rq;
rq = i915_gem_active_get_unlocked(&tl->last_request);
@@ -3276,7 +3276,7 @@ wait_for_timelines(struct drm_i915_private *i915,
/* restart after reacquiring the lock */
mutex_lock(>->mutex);
- tl = list_entry(>->list, typeof(*tl), link);
+ tl = list_entry(>->active_list, typeof(*tl), link);
}
mutex_unlock(>->mutex);
diff --git a/drivers/gpu/drm/i915/i915_reset.c b/drivers/gpu/drm/i915/i915_reset.c
index bd82f9b1043f..acf3c777e49d 100644
--- a/drivers/gpu/drm/i915/i915_reset.c
+++ b/drivers/gpu/drm/i915/i915_reset.c
@@ -856,7 +856,7 @@ bool i915_gem_unset_wedged(struct drm_i915_private *i915)
* No more can be submitted until we reset the wedged bit.
*/
mutex_lock(&i915->gt.timelines.mutex);
- list_for_each_entry(tl, &i915->gt.timelines.list, link) {
+ list_for_each_entry(tl, &i915->gt.timelines.active_list, link) {
struct i915_request *rq;
long timeout;
diff --git a/drivers/gpu/drm/i915/i915_timeline.c b/drivers/gpu/drm/i915/i915_timeline.c
index e4c11414a824..79838d89bdb9 100644
--- a/drivers/gpu/drm/i915/i915_timeline.c
+++ b/drivers/gpu/drm/i915/i915_timeline.c
@@ -120,7 +120,6 @@ int i915_timeline_init(struct drm_i915_private *i915,
const char *name,
struct i915_vma *hwsp)
{
- struct i915_gt_timelines *gt = &i915->gt.timelines;
void *vaddr;
/*
@@ -168,10 +167,6 @@ int i915_timeline_init(struct drm_i915_private *i915,
i915_syncmap_init(&timeline->sync);
- mutex_lock(>->mutex);
- list_add(&timeline->link, >->list);
- mutex_unlock(>->mutex);
-
return 0;
}
@@ -180,7 +175,7 @@ void i915_timelines_init(struct drm_i915_private *i915)
struct i915_gt_timelines *gt = &i915->gt.timelines;
mutex_init(>->mutex);
- INIT_LIST_HEAD(>->list);
+ INIT_LIST_HEAD(>->active_list);
spin_lock_init(>->hwsp_lock);
INIT_LIST_HEAD(>->hwsp_free_list);
@@ -189,6 +184,24 @@ void i915_timelines_init(struct drm_i915_private *i915)
i915_gem_shrinker_taints_mutex(i915, >->mutex);
}
+static void timeline_add_to_active(struct i915_timeline *tl)
+{
+ struct i915_gt_timelines *gt = &tl->i915->gt.timelines;
+
+ mutex_lock(>->mutex);
+ list_add(&tl->link, >->active_list);
+ mutex_unlock(>->mutex);
+}
+
+static void timeline_remove_from_active(struct i915_timeline *tl)
+{
+ struct i915_gt_timelines *gt = &tl->i915->gt.timelines;
+
+ mutex_lock(>->mutex);
+ list_del(&tl->link);
+ mutex_unlock(>->mutex);
+}
+
/**
* i915_timelines_park - called when the driver idles
* @i915: the drm_i915_private device
@@ -205,7 +218,7 @@ void i915_timelines_park(struct drm_i915_private *i915)
struct i915_timeline *timeline;
mutex_lock(>->mutex);
- list_for_each_entry(timeline, >->list, link) {
+ list_for_each_entry(timeline, >->active_list, link) {
/*
* All known fences are completed so we can scrap
* the current sync point tracking and start afresh,
@@ -219,15 +232,9 @@ void i915_timelines_park(struct drm_i915_private *i915)
void i915_timeline_fini(struct i915_timeline *timeline)
{
- struct i915_gt_timelines *gt = &timeline->i915->gt.timelines;
-
GEM_BUG_ON(timeline->pin_count);
GEM_BUG_ON(!list_empty(&timeline->requests));
- mutex_lock(>->mutex);
- list_del(&timeline->link);
- mutex_unlock(>->mutex);
-
i915_syncmap_free(&timeline->sync);
hwsp_free(timeline);
@@ -274,6 +281,8 @@ int i915_timeline_pin(struct i915_timeline *tl)
i915_ggtt_offset(tl->hwsp_ggtt) +
offset_in_page(tl->hwsp_offset);
+ timeline_add_to_active(tl);
+
return 0;
unpin:
@@ -287,6 +296,8 @@ void i915_timeline_unpin(struct i915_timeline *tl)
if (--tl->pin_count)
return;
+ timeline_remove_from_active(tl);
+
/*
* Since this timeline is idle, all bariers upon which we were waiting
* must also be complete and so we can discard the last used barriers
@@ -310,7 +321,7 @@ void i915_timelines_fini(struct drm_i915_private *i915)
{
struct i915_gt_timelines *gt = &i915->gt.timelines;
- GEM_BUG_ON(!list_empty(>->list));
+ GEM_BUG_ON(!list_empty(>->active_list));
GEM_BUG_ON(!list_empty(>->hwsp_free_list));
mutex_destroy(>->mutex);
--
2.20.1
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply related [flat|nested] 10+ messages in thread
* ✗ Fi.CI.CHECKPATCH: warning for series starting with [CI,1/6] drm/i915: Introduce concept of per-timeline (context) HWSP
2019-01-28 18:18 [CI 1/6] drm/i915: Introduce concept of per-timeline (context) HWSP Chris Wilson
` (4 preceding siblings ...)
2019-01-28 18:18 ` [CI 6/6] drm/i915: Track active timelines Chris Wilson
@ 2019-01-28 18:55 ` Patchwork
2019-01-28 18:59 ` ✗ Fi.CI.SPARSE: " Patchwork
` (2 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Patchwork @ 2019-01-28 18:55 UTC (permalink / raw)
To: Chris Wilson; +Cc: intel-gfx
== Series Details ==
Series: series starting with [CI,1/6] drm/i915: Introduce concept of per-timeline (context) HWSP
URL : https://patchwork.freedesktop.org/series/55856/
State : warning
== Summary ==
$ dim checkpatch origin/drm-tip
d5418c9092a6 drm/i915: Introduce concept of per-timeline (context) HWSP
7b660a9e3cd7 drm/i915: Enlarge vma->pin_count
6b1e92bf46ca drm/i915: Allocate a status page for each timeline
34006d56d0d6 drm/i915: Share per-timeline HWSP using a slab suballocator
-:85: CHECK:SPACING: No space is necessary after a cast
#85: FILE: drivers/gpu/drm/i915/i915_timeline.c:49:
+ BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);
total: 0 errors, 0 warnings, 1 checks, 408 lines checked
30cf3f265ea3 drm/i915: Track the context's seqno in its own timeline HWSP
84abe369e1be drm/i915: Track active timelines
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply [flat|nested] 10+ messages in thread
* ✗ Fi.CI.SPARSE: warning for series starting with [CI,1/6] drm/i915: Introduce concept of per-timeline (context) HWSP
2019-01-28 18:18 [CI 1/6] drm/i915: Introduce concept of per-timeline (context) HWSP Chris Wilson
` (5 preceding siblings ...)
2019-01-28 18:55 ` ✗ Fi.CI.CHECKPATCH: warning for series starting with [CI,1/6] drm/i915: Introduce concept of per-timeline (context) HWSP Patchwork
@ 2019-01-28 18:59 ` Patchwork
2019-01-28 19:17 ` ✓ Fi.CI.BAT: success " Patchwork
2019-01-29 2:06 ` ✓ Fi.CI.IGT: " Patchwork
8 siblings, 0 replies; 10+ messages in thread
From: Patchwork @ 2019-01-28 18:59 UTC (permalink / raw)
To: Chris Wilson; +Cc: intel-gfx
== Series Details ==
Series: series starting with [CI,1/6] drm/i915: Introduce concept of per-timeline (context) HWSP
URL : https://patchwork.freedesktop.org/series/55856/
State : warning
== Summary ==
$ dim sparse origin/drm-tip
Sparse version: v0.5.2
Commit: drm/i915: Introduce concept of per-timeline (context) HWSP
Okay!
Commit: drm/i915: Enlarge vma->pin_count
Okay!
Commit: drm/i915: Allocate a status page for each timeline
+./include/linux/mm.h:619:13: error: not a function <noident>
+./include/linux/mm.h:619:13: error: undefined identifier '__builtin_mul_overflow'
+./include/linux/mm.h:619:13: warning: call with no type!
Commit: drm/i915: Share per-timeline HWSP using a slab suballocator
-drivers/gpu/drm/i915/selftests/../i915_drv.h:3544:16: warning: expression using sizeof(void)
+drivers/gpu/drm/i915/selftests/../i915_drv.h:3548:16: warning: expression using sizeof(void)
+drivers/gpu/drm/i915/selftests/i915_timeline.c:89:38: warning: expression using sizeof(void)
+drivers/gpu/drm/i915/selftests/i915_timeline.c:89:38: warning: expression using sizeof(void)
+drivers/gpu/drm/i915/selftests/i915_timeline.c:92:44: warning: expression using sizeof(void)
+drivers/gpu/drm/i915/selftests/i915_timeline.c:92:44: warning: expression using sizeof(void)
+./include/linux/slab.h:664:13: error: undefined identifier '__builtin_mul_overflow'
+./include/linux/slab.h:664:13: warning: call with no type!
Commit: drm/i915: Track the context's seqno in its own timeline HWSP
Okay!
Commit: drm/i915: Track active timelines
Okay!
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply [flat|nested] 10+ messages in thread
* ✓ Fi.CI.BAT: success for series starting with [CI,1/6] drm/i915: Introduce concept of per-timeline (context) HWSP
2019-01-28 18:18 [CI 1/6] drm/i915: Introduce concept of per-timeline (context) HWSP Chris Wilson
` (6 preceding siblings ...)
2019-01-28 18:59 ` ✗ Fi.CI.SPARSE: " Patchwork
@ 2019-01-28 19:17 ` Patchwork
2019-01-29 2:06 ` ✓ Fi.CI.IGT: " Patchwork
8 siblings, 0 replies; 10+ messages in thread
From: Patchwork @ 2019-01-28 19:17 UTC (permalink / raw)
To: Chris Wilson; +Cc: intel-gfx
== Series Details ==
Series: series starting with [CI,1/6] drm/i915: Introduce concept of per-timeline (context) HWSP
URL : https://patchwork.freedesktop.org/series/55856/
State : success
== Summary ==
CI Bug Log - changes from CI_DRM_5497 -> Patchwork_12061
====================================================
Summary
-------
**SUCCESS**
No regressions found.
External URL: https://patchwork.freedesktop.org/api/1.0/series/55856/revisions/1/mbox/
Known issues
------------
Here are the changes found in Patchwork_12061 that come from known issues:
### IGT changes ###
#### Issues hit ####
* igt@i915_selftest@live_execlists:
- fi-apl-guc: PASS -> INCOMPLETE [fdo#103927]
* igt@kms_busy@basic-flip-b:
- fi-gdg-551: PASS -> FAIL [fdo#103182]
#### Possible fixes ####
* igt@kms_frontbuffer_tracking@basic:
- fi-byt-clapper: FAIL [fdo#103167] -> PASS
{name}: This element is suppressed. This means it is ignored when computing
the status of the difference (SUCCESS, WARNING, or FAILURE).
[fdo#103167]: https://bugs.freedesktop.org/show_bug.cgi?id=103167
[fdo#103182]: https://bugs.freedesktop.org/show_bug.cgi?id=103182
[fdo#103927]: https://bugs.freedesktop.org/show_bug.cgi?id=103927
[fdo#108622]: https://bugs.freedesktop.org/show_bug.cgi?id=108622
[fdo#109271]: https://bugs.freedesktop.org/show_bug.cgi?id=109271
Participating hosts (43 -> 41)
------------------------------
Additional (1): fi-hsw-4770r
Missing (3): fi-kbl-soraka fi-ilk-m540 fi-bsw-cyan
Build changes
-------------
* Linux: CI_DRM_5497 -> Patchwork_12061
CI_DRM_5497: 67aa6866708f2fa3a2ec914257cca94a872072ac @ git://anongit.freedesktop.org/gfx-ci/linux
IGT_4796: d1bd9c6ad6f3482bbccf4aa6417dd449e9efbe39 @ git://anongit.freedesktop.org/xorg/app/intel-gpu-tools
Patchwork_12061: 84abe369e1be51fe392e6a97143cd9377b8ea135 @ git://anongit.freedesktop.org/gfx-ci/linux
== Linux commits ==
84abe369e1be drm/i915: Track active timelines
30cf3f265ea3 drm/i915: Track the context's seqno in its own timeline HWSP
34006d56d0d6 drm/i915: Share per-timeline HWSP using a slab suballocator
6b1e92bf46ca drm/i915: Allocate a status page for each timeline
7b660a9e3cd7 drm/i915: Enlarge vma->pin_count
d5418c9092a6 drm/i915: Introduce concept of per-timeline (context) HWSP
== Logs ==
For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_12061/
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply [flat|nested] 10+ messages in thread
* ✓ Fi.CI.IGT: success for series starting with [CI,1/6] drm/i915: Introduce concept of per-timeline (context) HWSP
2019-01-28 18:18 [CI 1/6] drm/i915: Introduce concept of per-timeline (context) HWSP Chris Wilson
` (7 preceding siblings ...)
2019-01-28 19:17 ` ✓ Fi.CI.BAT: success " Patchwork
@ 2019-01-29 2:06 ` Patchwork
8 siblings, 0 replies; 10+ messages in thread
From: Patchwork @ 2019-01-29 2:06 UTC (permalink / raw)
To: Chris Wilson; +Cc: intel-gfx
== Series Details ==
Series: series starting with [CI,1/6] drm/i915: Introduce concept of per-timeline (context) HWSP
URL : https://patchwork.freedesktop.org/series/55856/
State : success
== Summary ==
CI Bug Log - changes from CI_DRM_5497_full -> Patchwork_12061_full
====================================================
Summary
-------
**SUCCESS**
No regressions found.
Known issues
------------
Here are the changes found in Patchwork_12061_full that come from known issues:
### IGT changes ###
#### Issues hit ####
* igt@kms_busy@extended-pageflip-modeset-hang-oldfb-render-c:
- shard-glk: PASS -> DMESG-WARN [fdo#107956]
* igt@kms_content_protection@atomic:
- shard-kbl: NOTRUN -> FAIL [fdo#108597]
- shard-apl: NOTRUN -> FAIL [fdo#108597]
* igt@kms_cursor_crc@cursor-128x128-dpms:
- shard-apl: PASS -> FAIL [fdo#103232]
- shard-glk: PASS -> FAIL [fdo#103232]
* igt@kms_flip@2x-dpms-vs-vblank-race:
- shard-hsw: PASS -> DMESG-FAIL [fdo#102614] / [fdo#103060]
* igt@kms_plane@plane-position-covered-pipe-a-planes:
- shard-glk: NOTRUN -> FAIL [fdo#103166]
- shard-apl: NOTRUN -> FAIL [fdo#103166]
* igt@kms_plane_multiple@atomic-pipe-a-tiling-x:
- shard-apl: PASS -> FAIL [fdo#103166] +1
* igt@kms_setmode@basic:
- shard-kbl: PASS -> FAIL [fdo#99912]
* igt@kms_sysfs_edid_timing:
- shard-apl: NOTRUN -> FAIL [fdo#100047]
- shard-kbl: NOTRUN -> FAIL [fdo#100047]
* igt@kms_universal_plane@universal-plane-pipe-c-functional:
- shard-glk: PASS -> FAIL [fdo#103166]
#### Possible fixes ####
* igt@gem_eio@in-flight-contexts-1us:
- shard-kbl: DMESG-WARN [fdo#109467] -> PASS +3
- shard-apl: DMESG-WARN [fdo#109467] -> PASS +3
* igt@gem_eio@in-flight-external:
- shard-glk: DMESG-WARN [fdo#109467] -> PASS +3
* igt@kms_busy@extended-modeset-hang-newfb-render-b:
- shard-snb: DMESG-WARN [fdo#107956] -> PASS
* igt@kms_color@pipe-a-ctm-max:
- shard-apl: FAIL [fdo#108147] -> PASS
* igt@kms_cursor_crc@cursor-64x21-sliding:
- shard-apl: FAIL [fdo#103232] -> PASS +1
* igt@kms_cursor_legacy@flip-vs-cursor-atomic-transitions-varying-size:
- shard-glk: INCOMPLETE [fdo#103359] / [k.org#198133] -> PASS
* igt@kms_plane_multiple@atomic-pipe-b-tiling-y:
- shard-glk: FAIL [fdo#103166] -> PASS
- shard-apl: FAIL [fdo#103166] -> PASS +2
{name}: This element is suppressed. This means it is ignored when computing
the status of the difference (SUCCESS, WARNING, or FAILURE).
[fdo#100047]: https://bugs.freedesktop.org/show_bug.cgi?id=100047
[fdo#102614]: https://bugs.freedesktop.org/show_bug.cgi?id=102614
[fdo#103060]: https://bugs.freedesktop.org/show_bug.cgi?id=103060
[fdo#103166]: https://bugs.freedesktop.org/show_bug.cgi?id=103166
[fdo#103232]: https://bugs.freedesktop.org/show_bug.cgi?id=103232
[fdo#103359]: https://bugs.freedesktop.org/show_bug.cgi?id=103359
[fdo#107956]: https://bugs.freedesktop.org/show_bug.cgi?id=107956
[fdo#108147]: https://bugs.freedesktop.org/show_bug.cgi?id=108147
[fdo#108597]: https://bugs.freedesktop.org/show_bug.cgi?id=108597
[fdo#109271]: https://bugs.freedesktop.org/show_bug.cgi?id=109271
[fdo#109278]: https://bugs.freedesktop.org/show_bug.cgi?id=109278
[fdo#109373]: https://bugs.freedesktop.org/show_bug.cgi?id=109373
[fdo#109467]: https://bugs.freedesktop.org/show_bug.cgi?id=109467
[fdo#99912]: https://bugs.freedesktop.org/show_bug.cgi?id=99912
[k.org#198133]: https://bugzilla.kernel.org/show_bug.cgi?id=198133
[k.org#202321]: https://bugzilla.kernel.org/show_bug.cgi?id=202321
Participating hosts (6 -> 5)
------------------------------
Missing (1): shard-skl
Build changes
-------------
* Linux: CI_DRM_5497 -> Patchwork_12061
CI_DRM_5497: 67aa6866708f2fa3a2ec914257cca94a872072ac @ git://anongit.freedesktop.org/gfx-ci/linux
IGT_4796: d1bd9c6ad6f3482bbccf4aa6417dd449e9efbe39 @ git://anongit.freedesktop.org/xorg/app/intel-gpu-tools
Patchwork_12061: 84abe369e1be51fe392e6a97143cd9377b8ea135 @ git://anongit.freedesktop.org/gfx-ci/linux
piglit_4509: fdc5a4ca11124ab8413c7988896eec4c97336694 @ git://anongit.freedesktop.org/piglit
== Logs ==
For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_12061/
_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
^ permalink raw reply [flat|nested] 10+ messages in thread
end of thread, other threads:[~2019-01-29 2:06 UTC | newest]
Thread overview: 10+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-01-28 18:18 [CI 1/6] drm/i915: Introduce concept of per-timeline (context) HWSP Chris Wilson
2019-01-28 18:18 ` [CI 2/6] drm/i915: Enlarge vma->pin_count Chris Wilson
2019-01-28 18:18 ` [CI 3/6] drm/i915: Allocate a status page for each timeline Chris Wilson
2019-01-28 18:18 ` [CI 4/6] drm/i915: Share per-timeline HWSP using a slab suballocator Chris Wilson
2019-01-28 18:18 ` [CI 5/6] drm/i915: Track the context's seqno in its own timeline HWSP Chris Wilson
2019-01-28 18:18 ` [CI 6/6] drm/i915: Track active timelines Chris Wilson
2019-01-28 18:55 ` ✗ Fi.CI.CHECKPATCH: warning for series starting with [CI,1/6] drm/i915: Introduce concept of per-timeline (context) HWSP Patchwork
2019-01-28 18:59 ` ✗ Fi.CI.SPARSE: " Patchwork
2019-01-28 19:17 ` ✓ Fi.CI.BAT: success " Patchwork
2019-01-29 2:06 ` ✓ Fi.CI.IGT: " Patchwork
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.