All of lore.kernel.org
 help / color / mirror / Atom feed
* [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing
@ 2021-02-05 10:49 Chris Wilson
  2021-02-05 11:15 ` [Intel-gfx] [PATCH v4] " Chris Wilson
                   ` (15 more replies)
  0 siblings, 16 replies; 21+ messages in thread
From: Chris Wilson @ 2021-02-05 10:49 UTC (permalink / raw)
  To: intel-gfx; +Cc: Chris Wilson

The heartbeat runs through a few phases that we expect to complete
within a certain number of heartbeat intervals. First we must submit the
heartbeat to the queue, and if the queue is occupied it may take a
couple of intervals before the heartbeat preempts the workload and is
submitted to HW. Once running on HW, completion is not instantaneous as
it may have to first reset the current workload before it itself runs
through the empty request and signals completion. As such, we know that
the heartbeat must take at least the preempt reset timeout and before we
have had a chance to reset the engine, we do not want to issue a global
reset ourselves (simply so that we only try to do one reset at a time
and not confuse ourselves by resetting twice and hitting an innocent.)

So by taking into consideration that once running the request must take
a finite amount of time, we can delay the final completion check to
accommodate that and avoid checking too early (before we've had a chance
to handle any engine resets required).

v2: Attach a callback to flush the work immediately upon the heartbeat
completion and insert the delay before the next.

Suggested-by: CQ Tang <cq.tang@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
 .../gpu/drm/i915/gt/intel_engine_heartbeat.c  | 91 +++++++++++++++++--
 drivers/gpu/drm/i915/gt/intel_engine_types.h  |  1 +
 .../drm/i915/gt/selftest_engine_heartbeat.c   | 50 +++++-----
 3 files changed, 106 insertions(+), 36 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
index 93741a65924a..5d91d42616c8 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
@@ -20,6 +20,18 @@
  * issue a reset -- in the hope that restores progress.
  */
 
+#define HEARTBEAT_COMPLETION 50u /* milliseconds */
+
+static long completion_timeout(const struct intel_engine_cs *engine)
+{
+	long timeout = HEARTBEAT_COMPLETION;
+
+	if (intel_engine_has_preempt_reset(engine))
+		timeout += READ_ONCE(engine->props.preempt_timeout_ms);
+
+	return msecs_to_jiffies(timeout);
+}
+
 static bool next_heartbeat(struct intel_engine_cs *engine)
 {
 	long delay;
@@ -29,6 +41,26 @@ static bool next_heartbeat(struct intel_engine_cs *engine)
 		return false;
 
 	delay = msecs_to_jiffies_timeout(delay);
+
+	/*
+	 * Once we submit a heartbeat to the HW, we know that it will take
+	 * at least a certain amount of time to complete. On a hanging system
+	 * it will first have to wait for the preempt reset timeout, and
+	 * then it will take some time for the reset to resume with the
+	 * heartbeat and for it to complete. So once we have submitted the
+	 * heartbeat to HW, we can wait a while longer before declaring the
+	 * engine stuck and forcing a reset ourselves. If we do a reset
+	 * and the engine is also doing a reset, it is possible that we
+	 * reset the engine twice, harming an innocent.
+	 *
+	 * Before we have sumitted the heartbeat, we do not want to change
+	 * the interval as we to promote the heartbeat and trigger preemption
+	 * in a deterministic time frame.
+	 */
+	if (engine->heartbeat.systole &&
+	    i915_request_is_active(engine->heartbeat.systole))
+		delay = max(delay, completion_timeout(engine));
+
 	if (delay >= HZ)
 		delay = round_jiffies_up_relative(delay);
 	mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, delay + 1);
@@ -48,12 +80,51 @@ heartbeat_create(struct intel_context *ce, gfp_t gfp)
 	return rq;
 }
 
+static void defibrillator(struct dma_fence *f, struct dma_fence_cb *cb)
+{
+	struct intel_engine_cs *engine =
+		container_of(cb, typeof(*engine), heartbeat.cb);
+
+	if (READ_ONCE(engine->heartbeat.systole))
+		mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, 0);
+}
+
+static void
+untrack_heartbeat(struct intel_engine_cs *engine)
+{
+	struct i915_request *rq;
+
+	rq = fetch_and_zero(&engine->heartbeat.systole);
+	if (!rq)
+		return;
+
+	ENGINE_TRACE(engine, "heartbeat completed: %llx:%lld\n",
+		     rq->fence.context, rq->fence.seqno);
+
+	dma_fence_remove_callback(&rq->fence, &engine->heartbeat.cb);
+	i915_request_put(rq);
+}
+
+static void
+track_heartbeat(struct intel_engine_cs *engine, struct i915_request *rq)
+{
+	ENGINE_TRACE(engine, "heartbeat started: %llx:%lld\n",
+		     rq->fence.context, rq->fence.seqno);
+
+	dma_fence_add_callback(&rq->fence,
+			       &engine->heartbeat.cb,
+			       defibrillator);
+	engine->heartbeat.systole = i915_request_get(rq);
+	if (!next_heartbeat(engine))
+		untrack_heartbeat(engine);
+}
+
 static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq)
 {
 	engine->wakeref_serial = READ_ONCE(engine->serial) + 1;
 	i915_request_add_active_barriers(rq);
 	if (!engine->heartbeat.systole && intel_engine_has_heartbeat(engine))
-		engine->heartbeat.systole = i915_request_get(rq);
+		track_heartbeat(engine, rq);
 }
 
 static void heartbeat_commit(struct i915_request *rq,
@@ -91,10 +162,8 @@ static void heartbeat(struct work_struct *wrk)
 	intel_engine_flush_submission(engine);
 
 	rq = engine->heartbeat.systole;
-	if (rq && i915_request_completed(rq)) {
-		i915_request_put(rq);
-		engine->heartbeat.systole = NULL;
-	}
+	if (rq && i915_request_completed(rq))
+		untrack_heartbeat(engine);
 
 	if (!intel_engine_pm_get_if_awake(engine))
 		return;
@@ -151,6 +220,11 @@ static void heartbeat(struct work_struct *wrk)
 		goto out;
 	}
 
+	/* Just completed one heartbeat, wait a tick before the next */
+	if (rq)
+		goto out;
+
+	/* The engine is parking. We can rest until the next user */
 	serial = READ_ONCE(engine->serial);
 	if (engine->wakeref_serial == serial)
 		goto out;
@@ -174,8 +248,9 @@ static void heartbeat(struct work_struct *wrk)
 unlock:
 	mutex_unlock(&ce->timeline->mutex);
 out:
+	intel_engine_flush_submission(engine);
 	if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+		untrack_heartbeat(engine);
 	intel_engine_pm_put(engine);
 }
 
@@ -189,8 +264,8 @@ void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine)
 
 void intel_engine_park_heartbeat(struct intel_engine_cs *engine)
 {
-	if (cancel_delayed_work(&engine->heartbeat.work))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+	while (cancel_delayed_work(&engine->heartbeat.work))
+		untrack_heartbeat(engine); /* completion may rearm work */
 }
 
 void intel_engine_init_heartbeat(struct intel_engine_cs *engine)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 7159f9575e65..4956594c8b93 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -347,6 +347,7 @@ struct intel_engine_cs {
 	struct {
 		struct delayed_work work;
 		struct i915_request *systole;
+		struct dma_fence_cb cb;
 		unsigned long blocked;
 	} heartbeat;
 
diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
index b2c369317bf1..429cfca6fb28 100644
--- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
@@ -202,18 +202,13 @@ static int cmp_u32(const void *_a, const void *_b)
 
 static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 {
-	const unsigned int error_threshold = max(20000u, jiffies_to_usecs(6));
-	struct intel_context *ce;
-	struct i915_request *rq;
-	ktime_t t0, t1;
+	const unsigned int error_threshold =
+		max(3 * HEARTBEAT_COMPLETION * 1000, jiffies_to_usecs(6));
+	struct intel_context *ce = engine->kernel_context;
 	u32 times[5];
 	int err;
 	int i;
 
-	ce = intel_context_create(engine);
-	if (IS_ERR(ce))
-		return PTR_ERR(ce);
-
 	intel_engine_pm_get(engine);
 
 	err = intel_engine_set_heartbeat(engine, 1);
@@ -221,28 +216,27 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 		goto err_pm;
 
 	for (i = 0; i < ARRAY_SIZE(times); i++) {
-		do {
-			/* Manufacture a tick */
-			intel_engine_park_heartbeat(engine);
-			GEM_BUG_ON(engine->heartbeat.systole);
-			engine->serial++; /*  pretend we are not idle! */
-			intel_engine_unpark_heartbeat(engine);
+		struct i915_sched_attr attr = { .priority = I915_PRIORITY_MIN };
+		struct i915_request *rq;
+		ktime_t t0, t1;
 
-			flush_delayed_work(&engine->heartbeat.work);
-			if (!delayed_work_pending(&engine->heartbeat.work)) {
-				pr_err("%s: heartbeat %d did not start\n",
-				       engine->name, i);
-				err = -EINVAL;
-				goto err_pm;
-			}
+		intel_engine_park_heartbeat(engine);
+		GEM_BUG_ON(engine->heartbeat.systole);
 
-			rcu_read_lock();
-			rq = READ_ONCE(engine->heartbeat.systole);
-			if (rq)
-				rq = i915_request_get_rcu(rq);
-			rcu_read_unlock();
-		} while (!rq);
+		/* Manufacture a tick */
+		mutex_lock(&ce->timeline->mutex);
+		rq = heartbeat_create(ce, GFP_KERNEL);
+		if (!IS_ERR(rq)) {
+			i915_request_get(rq);
+			heartbeat_commit(rq, &attr);
+		}
+		mutex_unlock(&ce->timeline->mutex);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto err_reset;
+		}
 
+		/* Time how long before the heartbeat monitor checks */
 		t0 = ktime_get();
 		while (rq == READ_ONCE(engine->heartbeat.systole))
 			yield(); /* work is on the local cpu! */
@@ -275,10 +269,10 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 		err = -EINVAL;
 	}
 
+err_reset:
 	reset_heartbeat(engine);
 err_pm:
 	intel_engine_pm_put(engine);
-	intel_context_put(ce);
 	return err;
 }
 
-- 
2.20.1

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply related	[flat|nested] 21+ messages in thread

* [Intel-gfx] [PATCH v4] drm/i915/gt: Ratelimit heartbeat completion probing
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
@ 2021-02-05 11:15 ` Chris Wilson
  2021-02-05 13:15 ` [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev5) Patchwork
                   ` (14 subsequent siblings)
  15 siblings, 0 replies; 21+ messages in thread
From: Chris Wilson @ 2021-02-05 11:15 UTC (permalink / raw)
  To: intel-gfx; +Cc: Chris Wilson

The heartbeat runs through a few phases that we expect to complete
within a certain number of heartbeat intervals. First we must submit the
heartbeat to the queue, and if the queue is occupied it may take a
couple of intervals before the heartbeat preempts the workload and is
submitted to HW. Once running on HW, completion is not instantaneous as
it may have to first reset the current workload before it itself runs
through the empty request and signals completion. As such, we know that
the heartbeat must take at least the preempt reset timeout and before we
have had a chance to reset the engine, we do not want to issue a global
reset ourselves (simply so that we only try to do one reset at a time
and not confuse ourselves by resetting twice and hitting an innocent.)

So by taking into consideration that once running the request must take
a finite amount of time, we can delay the final completion check to
accommodate that and avoid checking too early (before we've had a chance
to handle any engine resets required).

v2: Attach a callback to flush the work immediately upon the heartbeat
completion and insert the delay before the next.

Suggested-by: CQ Tang <cq.tang@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
 .../gpu/drm/i915/gt/intel_engine_heartbeat.c  | 91 +++++++++++++++++--
 drivers/gpu/drm/i915/gt/intel_engine_types.h  |  1 +
 .../drm/i915/gt/selftest_engine_heartbeat.c   | 50 +++++-----
 drivers/gpu/drm/i915/gt/selftest_execlists.c  |  5 +-
 4 files changed, 110 insertions(+), 37 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
index 93741a65924a..5d91d42616c8 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
@@ -20,6 +20,18 @@
  * issue a reset -- in the hope that restores progress.
  */
 
+#define HEARTBEAT_COMPLETION 50u /* milliseconds */
+
+static long completion_timeout(const struct intel_engine_cs *engine)
+{
+	long timeout = HEARTBEAT_COMPLETION;
+
+	if (intel_engine_has_preempt_reset(engine))
+		timeout += READ_ONCE(engine->props.preempt_timeout_ms);
+
+	return msecs_to_jiffies(timeout);
+}
+
 static bool next_heartbeat(struct intel_engine_cs *engine)
 {
 	long delay;
@@ -29,6 +41,26 @@ static bool next_heartbeat(struct intel_engine_cs *engine)
 		return false;
 
 	delay = msecs_to_jiffies_timeout(delay);
+
+	/*
+	 * Once we submit a heartbeat to the HW, we know that it will take
+	 * at least a certain amount of time to complete. On a hanging system
+	 * it will first have to wait for the preempt reset timeout, and
+	 * then it will take some time for the reset to resume with the
+	 * heartbeat and for it to complete. So once we have submitted the
+	 * heartbeat to HW, we can wait a while longer before declaring the
+	 * engine stuck and forcing a reset ourselves. If we do a reset
+	 * and the engine is also doing a reset, it is possible that we
+	 * reset the engine twice, harming an innocent.
+	 *
+	 * Before we have sumitted the heartbeat, we do not want to change
+	 * the interval as we to promote the heartbeat and trigger preemption
+	 * in a deterministic time frame.
+	 */
+	if (engine->heartbeat.systole &&
+	    i915_request_is_active(engine->heartbeat.systole))
+		delay = max(delay, completion_timeout(engine));
+
 	if (delay >= HZ)
 		delay = round_jiffies_up_relative(delay);
 	mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, delay + 1);
@@ -48,12 +80,51 @@ heartbeat_create(struct intel_context *ce, gfp_t gfp)
 	return rq;
 }
 
+static void defibrillator(struct dma_fence *f, struct dma_fence_cb *cb)
+{
+	struct intel_engine_cs *engine =
+		container_of(cb, typeof(*engine), heartbeat.cb);
+
+	if (READ_ONCE(engine->heartbeat.systole))
+		mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, 0);
+}
+
+static void
+untrack_heartbeat(struct intel_engine_cs *engine)
+{
+	struct i915_request *rq;
+
+	rq = fetch_and_zero(&engine->heartbeat.systole);
+	if (!rq)
+		return;
+
+	ENGINE_TRACE(engine, "heartbeat completed: %llx:%lld\n",
+		     rq->fence.context, rq->fence.seqno);
+
+	dma_fence_remove_callback(&rq->fence, &engine->heartbeat.cb);
+	i915_request_put(rq);
+}
+
+static void
+track_heartbeat(struct intel_engine_cs *engine, struct i915_request *rq)
+{
+	ENGINE_TRACE(engine, "heartbeat started: %llx:%lld\n",
+		     rq->fence.context, rq->fence.seqno);
+
+	dma_fence_add_callback(&rq->fence,
+			       &engine->heartbeat.cb,
+			       defibrillator);
+	engine->heartbeat.systole = i915_request_get(rq);
+	if (!next_heartbeat(engine))
+		untrack_heartbeat(engine);
+}
+
 static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq)
 {
 	engine->wakeref_serial = READ_ONCE(engine->serial) + 1;
 	i915_request_add_active_barriers(rq);
 	if (!engine->heartbeat.systole && intel_engine_has_heartbeat(engine))
-		engine->heartbeat.systole = i915_request_get(rq);
+		track_heartbeat(engine, rq);
 }
 
 static void heartbeat_commit(struct i915_request *rq,
@@ -91,10 +162,8 @@ static void heartbeat(struct work_struct *wrk)
 	intel_engine_flush_submission(engine);
 
 	rq = engine->heartbeat.systole;
-	if (rq && i915_request_completed(rq)) {
-		i915_request_put(rq);
-		engine->heartbeat.systole = NULL;
-	}
+	if (rq && i915_request_completed(rq))
+		untrack_heartbeat(engine);
 
 	if (!intel_engine_pm_get_if_awake(engine))
 		return;
@@ -151,6 +220,11 @@ static void heartbeat(struct work_struct *wrk)
 		goto out;
 	}
 
+	/* Just completed one heartbeat, wait a tick before the next */
+	if (rq)
+		goto out;
+
+	/* The engine is parking. We can rest until the next user */
 	serial = READ_ONCE(engine->serial);
 	if (engine->wakeref_serial == serial)
 		goto out;
@@ -174,8 +248,9 @@ static void heartbeat(struct work_struct *wrk)
 unlock:
 	mutex_unlock(&ce->timeline->mutex);
 out:
+	intel_engine_flush_submission(engine);
 	if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+		untrack_heartbeat(engine);
 	intel_engine_pm_put(engine);
 }
 
@@ -189,8 +264,8 @@ void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine)
 
 void intel_engine_park_heartbeat(struct intel_engine_cs *engine)
 {
-	if (cancel_delayed_work(&engine->heartbeat.work))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+	while (cancel_delayed_work(&engine->heartbeat.work))
+		untrack_heartbeat(engine); /* completion may rearm work */
 }
 
 void intel_engine_init_heartbeat(struct intel_engine_cs *engine)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 7159f9575e65..4956594c8b93 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -347,6 +347,7 @@ struct intel_engine_cs {
 	struct {
 		struct delayed_work work;
 		struct i915_request *systole;
+		struct dma_fence_cb cb;
 		unsigned long blocked;
 	} heartbeat;
 
diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
index b2c369317bf1..429cfca6fb28 100644
--- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
@@ -202,18 +202,13 @@ static int cmp_u32(const void *_a, const void *_b)
 
 static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 {
-	const unsigned int error_threshold = max(20000u, jiffies_to_usecs(6));
-	struct intel_context *ce;
-	struct i915_request *rq;
-	ktime_t t0, t1;
+	const unsigned int error_threshold =
+		max(3 * HEARTBEAT_COMPLETION * 1000, jiffies_to_usecs(6));
+	struct intel_context *ce = engine->kernel_context;
 	u32 times[5];
 	int err;
 	int i;
 
-	ce = intel_context_create(engine);
-	if (IS_ERR(ce))
-		return PTR_ERR(ce);
-
 	intel_engine_pm_get(engine);
 
 	err = intel_engine_set_heartbeat(engine, 1);
@@ -221,28 +216,27 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 		goto err_pm;
 
 	for (i = 0; i < ARRAY_SIZE(times); i++) {
-		do {
-			/* Manufacture a tick */
-			intel_engine_park_heartbeat(engine);
-			GEM_BUG_ON(engine->heartbeat.systole);
-			engine->serial++; /*  pretend we are not idle! */
-			intel_engine_unpark_heartbeat(engine);
+		struct i915_sched_attr attr = { .priority = I915_PRIORITY_MIN };
+		struct i915_request *rq;
+		ktime_t t0, t1;
 
-			flush_delayed_work(&engine->heartbeat.work);
-			if (!delayed_work_pending(&engine->heartbeat.work)) {
-				pr_err("%s: heartbeat %d did not start\n",
-				       engine->name, i);
-				err = -EINVAL;
-				goto err_pm;
-			}
+		intel_engine_park_heartbeat(engine);
+		GEM_BUG_ON(engine->heartbeat.systole);
 
-			rcu_read_lock();
-			rq = READ_ONCE(engine->heartbeat.systole);
-			if (rq)
-				rq = i915_request_get_rcu(rq);
-			rcu_read_unlock();
-		} while (!rq);
+		/* Manufacture a tick */
+		mutex_lock(&ce->timeline->mutex);
+		rq = heartbeat_create(ce, GFP_KERNEL);
+		if (!IS_ERR(rq)) {
+			i915_request_get(rq);
+			heartbeat_commit(rq, &attr);
+		}
+		mutex_unlock(&ce->timeline->mutex);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto err_reset;
+		}
 
+		/* Time how long before the heartbeat monitor checks */
 		t0 = ktime_get();
 		while (rq == READ_ONCE(engine->heartbeat.systole))
 			yield(); /* work is on the local cpu! */
@@ -275,10 +269,10 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 		err = -EINVAL;
 	}
 
+err_reset:
 	reset_heartbeat(engine);
 err_pm:
 	intel_engine_pm_put(engine);
-	intel_context_put(ce);
 	return err;
 }
 
diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c
index 7bab147e4421..4f7298855278 100644
--- a/drivers/gpu/drm/i915/gt/selftest_execlists.c
+++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c
@@ -2324,13 +2324,16 @@ static int __cancel_fail(struct live_preempt_cancel *arg)
 	del_timer_sync(&engine->execlists.preempt);
 	intel_engine_flush_submission(engine);
 
+	engine->props.preempt_timeout_ms = 0;
 	cancel_reset_timeout(engine);
 
-	/* after failure, require heartbeats to reset device */
+	/* after failure, require fast heartbeats to reset device */
 	intel_engine_set_heartbeat(engine, 1);
 	err = wait_for_reset(engine, rq, HZ / 2);
 	intel_engine_set_heartbeat(engine,
 				   engine->defaults.heartbeat_interval_ms);
+
+	engine->props.preempt_timeout_ms = engine->defaults.preempt_timeout_ms;
 	if (err) {
 		pr_err("Cancelled inflight0 request did not reset\n");
 		goto out;
-- 
2.20.1

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply related	[flat|nested] 21+ messages in thread

* [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev5)
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
  2021-02-05 11:15 ` [Intel-gfx] [PATCH v4] " Chris Wilson
@ 2021-02-05 13:15 ` Patchwork
  2021-02-05 14:22 ` [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (13 subsequent siblings)
  15 siblings, 0 replies; 21+ messages in thread
From: Patchwork @ 2021-02-05 13:15 UTC (permalink / raw)
  To: Chris Wilson; +Cc: intel-gfx


[-- Attachment #1.1: Type: text/plain, Size: 4221 bytes --]

== Series Details ==

Series: drm/i915/gt: Ratelimit heartbeat completion probing (rev5)
URL   : https://patchwork.freedesktop.org/series/86665/
State : success

== Summary ==

CI Bug Log - changes from CI_DRM_9737 -> Patchwork_19603
====================================================

Summary
-------

  **SUCCESS**

  No regressions found.

  External URL: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19603/index.html

Known issues
------------

  Here are the changes found in Patchwork_19603 that come from known issues:

### IGT changes ###

#### Issues hit ####

  * igt@i915_selftest@live@gt_heartbeat:
    - fi-tgl-y:           [PASS][1] -> [DMESG-FAIL][2] ([i915#2601])
   [1]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9737/fi-tgl-y/igt@i915_selftest@live@gt_heartbeat.html
   [2]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19603/fi-tgl-y/igt@i915_selftest@live@gt_heartbeat.html
    - fi-kbl-r:           [PASS][3] -> [INCOMPLETE][4] ([i915#2782] / [i915#2853])
   [3]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9737/fi-kbl-r/igt@i915_selftest@live@gt_heartbeat.html
   [4]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19603/fi-kbl-r/igt@i915_selftest@live@gt_heartbeat.html
    - fi-cfl-guc:         [PASS][5] -> [INCOMPLETE][6] ([i915#2853])
   [5]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9737/fi-cfl-guc/igt@i915_selftest@live@gt_heartbeat.html
   [6]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19603/fi-cfl-guc/igt@i915_selftest@live@gt_heartbeat.html

  * igt@kms_chamelium@dp-crc-fast:
    - fi-kbl-7500u:       [PASS][7] -> [FAIL][8] ([i915#1372])
   [7]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9737/fi-kbl-7500u/igt@kms_chamelium@dp-crc-fast.html
   [8]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19603/fi-kbl-7500u/igt@kms_chamelium@dp-crc-fast.html

  * igt@prime_vgem@basic-fence-flip:
    - fi-tgl-y:           [PASS][9] -> [DMESG-WARN][10] ([i915#402])
   [9]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9737/fi-tgl-y/igt@prime_vgem@basic-fence-flip.html
   [10]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19603/fi-tgl-y/igt@prime_vgem@basic-fence-flip.html

  
#### Possible fixes ####

  * igt@gem_exec_suspend@basic-s3:
    - fi-tgl-u2:          [FAIL][11] ([i915#1888]) -> [PASS][12]
   [11]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9737/fi-tgl-u2/igt@gem_exec_suspend@basic-s3.html
   [12]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19603/fi-tgl-u2/igt@gem_exec_suspend@basic-s3.html

  * igt@prime_self_import@basic-with_one_bo_two_files:
    - fi-tgl-y:           [DMESG-WARN][13] ([i915#402]) -> [PASS][14] +1 similar issue
   [13]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9737/fi-tgl-y/igt@prime_self_import@basic-with_one_bo_two_files.html
   [14]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19603/fi-tgl-y/igt@prime_self_import@basic-with_one_bo_two_files.html

  
  {name}: This element is suppressed. This means it is ignored when computing
          the status of the difference (SUCCESS, WARNING, or FAILURE).

  [i915#1372]: https://gitlab.freedesktop.org/drm/intel/issues/1372
  [i915#1888]: https://gitlab.freedesktop.org/drm/intel/issues/1888
  [i915#2601]: https://gitlab.freedesktop.org/drm/intel/issues/2601
  [i915#2782]: https://gitlab.freedesktop.org/drm/intel/issues/2782
  [i915#2853]: https://gitlab.freedesktop.org/drm/intel/issues/2853
  [i915#402]: https://gitlab.freedesktop.org/drm/intel/issues/402


Participating hosts (43 -> 38)
------------------------------

  Missing    (5): fi-jsl-1 fi-ilk-m540 fi-bsw-cyan fi-icl-u2 fi-bdw-samus 


Build changes
-------------

  * Linux: CI_DRM_9737 -> Patchwork_19603

  CI-20190529: 20190529
  CI_DRM_9737: 8fef45d7241af38b7d68a8ad3b11ce8ab38b4491 @ git://anongit.freedesktop.org/gfx-ci/linux
  IGT_5992: b781a32b06a0173a71b4e1ac30d18dd7164a67c4 @ git://anongit.freedesktop.org/xorg/app/intel-gpu-tools
  Patchwork_19603: 3ba7586ed485d5ebb4f5b9e7158372910630218d @ git://anongit.freedesktop.org/gfx-ci/linux


== Linux commits ==

3ba7586ed485 drm/i915/gt: Ratelimit heartbeat completion probing

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19603/index.html

[-- Attachment #1.2: Type: text/html, Size: 5114 bytes --]

[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply	[flat|nested] 21+ messages in thread

* [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
  2021-02-05 11:15 ` [Intel-gfx] [PATCH v4] " Chris Wilson
  2021-02-05 13:15 ` [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev5) Patchwork
@ 2021-02-05 14:22 ` Chris Wilson
  2021-02-05 14:32 ` Chris Wilson
                   ` (12 subsequent siblings)
  15 siblings, 0 replies; 21+ messages in thread
From: Chris Wilson @ 2021-02-05 14:22 UTC (permalink / raw)
  To: intel-gfx; +Cc: Chris Wilson

The heartbeat runs through a few phases that we expect to complete
within a certain number of heartbeat intervals. First we must submit the
heartbeat to the queue, and if the queue is occupied it may take a
couple of intervals before the heartbeat preempts the workload and is
submitted to HW. Once running on HW, completion is not instantaneous as
it may have to first reset the current workload before it itself runs
through the empty request and signals completion. As such, we know that
the heartbeat must take at least the preempt reset timeout and before we
have had a chance to reset the engine, we do not want to issue a global
reset ourselves (simply so that we only try to do one reset at a time
and not confuse ourselves by resetting twice and hitting an innocent.)

So by taking into consideration that once running the request must take
a finite amount of time, we can delay the final completion check to
accommodate that and avoid checking too early (before we've had a chance
to handle any engine resets required).

v2: Attach a callback to flush the work immediately upon the heartbeat
completion and insert the delay before the next.

Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/2853
Suggested-by: CQ Tang <cq.tang@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
 .../gpu/drm/i915/gt/intel_engine_heartbeat.c  | 93 +++++++++++++++++--
 drivers/gpu/drm/i915/gt/intel_engine_types.h  |  1 +
 .../drm/i915/gt/selftest_engine_heartbeat.c   | 62 ++++++-------
 drivers/gpu/drm/i915/gt/selftest_execlists.c  |  5 +-
 4 files changed, 116 insertions(+), 45 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
index 93741a65924a..9089689c84eb 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
@@ -20,6 +20,18 @@
  * issue a reset -- in the hope that restores progress.
  */
 
+#define HEARTBEAT_COMPLETION 50u /* milliseconds */
+
+static long completion_timeout(const struct intel_engine_cs *engine)
+{
+	long timeout = HEARTBEAT_COMPLETION;
+
+	if (intel_engine_has_preempt_reset(engine))
+		timeout += READ_ONCE(engine->props.preempt_timeout_ms);
+
+	return msecs_to_jiffies(timeout);
+}
+
 static bool next_heartbeat(struct intel_engine_cs *engine)
 {
 	long delay;
@@ -29,6 +41,26 @@ static bool next_heartbeat(struct intel_engine_cs *engine)
 		return false;
 
 	delay = msecs_to_jiffies_timeout(delay);
+
+	/*
+	 * Once we submit a heartbeat to the HW, we know that it will take
+	 * at least a certain amount of time to complete. On a hanging system
+	 * it will first have to wait for the preempt reset timeout, and
+	 * then it will take some time for the reset to resume with the
+	 * heartbeat and for it to complete. So once we have submitted the
+	 * heartbeat to HW, we can wait a while longer before declaring the
+	 * engine stuck and forcing a reset ourselves. If we do a reset
+	 * and the engine is also doing a reset, it is possible that we
+	 * reset the engine twice, harming an innocent.
+	 *
+	 * Before we have sumitted the heartbeat, we do not want to change
+	 * the interval as we to promote the heartbeat and trigger preemption
+	 * in a deterministic time frame.
+	 */
+	if (engine->heartbeat.systole &&
+	    i915_request_is_active(engine->heartbeat.systole))
+		delay = max(delay, completion_timeout(engine));
+
 	if (delay >= HZ)
 		delay = round_jiffies_up_relative(delay);
 	mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, delay + 1);
@@ -48,12 +80,51 @@ heartbeat_create(struct intel_context *ce, gfp_t gfp)
 	return rq;
 }
 
+static void defibrillator(struct dma_fence *f, struct dma_fence_cb *cb)
+{
+	struct intel_engine_cs *engine =
+		container_of(cb, typeof(*engine), heartbeat.cb);
+
+	if (READ_ONCE(engine->heartbeat.systole))
+		mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, 0);
+}
+
+static void
+untrack_heartbeat(struct intel_engine_cs *engine)
+{
+	struct i915_request *rq;
+
+	rq = fetch_and_zero(&engine->heartbeat.systole);
+	if (!rq)
+		return;
+
+	ENGINE_TRACE(engine, "heartbeat completed: %llx:%lld\n",
+		     rq->fence.context, rq->fence.seqno);
+
+	dma_fence_remove_callback(&rq->fence, &engine->heartbeat.cb);
+	i915_request_put(rq);
+}
+
+static void
+track_heartbeat(struct intel_engine_cs *engine, struct i915_request *rq)
+{
+	ENGINE_TRACE(engine, "heartbeat started: %llx:%lld\n",
+		     rq->fence.context, rq->fence.seqno);
+
+	dma_fence_add_callback(&rq->fence,
+			       &engine->heartbeat.cb,
+			       defibrillator);
+	engine->heartbeat.systole = i915_request_get(rq);
+	if (!next_heartbeat(engine))
+		untrack_heartbeat(engine);
+}
+
 static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq)
 {
 	engine->wakeref_serial = READ_ONCE(engine->serial) + 1;
 	i915_request_add_active_barriers(rq);
 	if (!engine->heartbeat.systole && intel_engine_has_heartbeat(engine))
-		engine->heartbeat.systole = i915_request_get(rq);
+		track_heartbeat(engine, rq);
 }
 
 static void heartbeat_commit(struct i915_request *rq,
@@ -91,10 +162,8 @@ static void heartbeat(struct work_struct *wrk)
 	intel_engine_flush_submission(engine);
 
 	rq = engine->heartbeat.systole;
-	if (rq && i915_request_completed(rq)) {
-		i915_request_put(rq);
-		engine->heartbeat.systole = NULL;
-	}
+	if (rq && i915_request_completed(rq))
+		untrack_heartbeat(engine);
 
 	if (!intel_engine_pm_get_if_awake(engine))
 		return;
@@ -151,6 +220,11 @@ static void heartbeat(struct work_struct *wrk)
 		goto out;
 	}
 
+	/* Just completed one heartbeat, wait a tick before the next */
+	if (rq)
+		goto out;
+
+	/* The engine is parking. We can rest until the next user */
 	serial = READ_ONCE(engine->serial);
 	if (engine->wakeref_serial == serial)
 		goto out;
@@ -174,8 +248,9 @@ static void heartbeat(struct work_struct *wrk)
 unlock:
 	mutex_unlock(&ce->timeline->mutex);
 out:
+	intel_engine_flush_submission(engine);
 	if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+		untrack_heartbeat(engine);
 	intel_engine_pm_put(engine);
 }
 
@@ -189,8 +264,10 @@ void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine)
 
 void intel_engine_park_heartbeat(struct intel_engine_cs *engine)
 {
-	if (cancel_delayed_work(&engine->heartbeat.work))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+       	/* completion may rearm work */
+	while (cancel_delayed_work(&engine->heartbeat.work))
+		untrack_heartbeat(engine);
+	untrack_heartbeat(engine);
 }
 
 void intel_engine_init_heartbeat(struct intel_engine_cs *engine)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 7159f9575e65..4956594c8b93 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -347,6 +347,7 @@ struct intel_engine_cs {
 	struct {
 		struct delayed_work work;
 		struct i915_request *systole;
+		struct dma_fence_cb cb;
 		unsigned long blocked;
 	} heartbeat;
 
diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
index b2c369317bf1..d97294e7c204 100644
--- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
@@ -202,18 +202,13 @@ static int cmp_u32(const void *_a, const void *_b)
 
 static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 {
-	const unsigned int error_threshold = max(20000u, jiffies_to_usecs(6));
-	struct intel_context *ce;
-	struct i915_request *rq;
-	ktime_t t0, t1;
+	const unsigned int error_threshold =
+		max(3 * HEARTBEAT_COMPLETION * 1000, jiffies_to_usecs(6));
+	struct intel_context *ce = engine->kernel_context;
 	u32 times[5];
 	int err;
 	int i;
 
-	ce = intel_context_create(engine);
-	if (IS_ERR(ce))
-		return PTR_ERR(ce);
-
 	intel_engine_pm_get(engine);
 
 	err = intel_engine_set_heartbeat(engine, 1);
@@ -221,28 +216,27 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 		goto err_pm;
 
 	for (i = 0; i < ARRAY_SIZE(times); i++) {
-		do {
-			/* Manufacture a tick */
-			intel_engine_park_heartbeat(engine);
-			GEM_BUG_ON(engine->heartbeat.systole);
-			engine->serial++; /*  pretend we are not idle! */
-			intel_engine_unpark_heartbeat(engine);
+		struct i915_sched_attr attr = { .priority = I915_PRIORITY_MIN };
+		struct i915_request *rq;
+		ktime_t t0, t1;
 
-			flush_delayed_work(&engine->heartbeat.work);
-			if (!delayed_work_pending(&engine->heartbeat.work)) {
-				pr_err("%s: heartbeat %d did not start\n",
-				       engine->name, i);
-				err = -EINVAL;
-				goto err_pm;
-			}
+		intel_engine_park_heartbeat(engine);
+		GEM_BUG_ON(engine->heartbeat.systole);
 
-			rcu_read_lock();
-			rq = READ_ONCE(engine->heartbeat.systole);
-			if (rq)
-				rq = i915_request_get_rcu(rq);
-			rcu_read_unlock();
-		} while (!rq);
+		/* Manufacture a tick */
+		mutex_lock(&ce->timeline->mutex);
+		rq = heartbeat_create(ce, GFP_KERNEL);
+		if (!IS_ERR(rq)) {
+			i915_request_get(rq);
+			heartbeat_commit(rq, &attr);
+		}
+		mutex_unlock(&ce->timeline->mutex);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto err_reset;
+		}
 
+		/* Time how long before the heartbeat monitor checks */
 		t0 = ktime_get();
 		while (rq == READ_ONCE(engine->heartbeat.systole))
 			yield(); /* work is on the local cpu! */
@@ -275,10 +269,10 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 		err = -EINVAL;
 	}
 
+err_reset:
 	reset_heartbeat(engine);
 err_pm:
 	intel_engine_pm_put(engine);
-	intel_context_put(ce);
 	return err;
 }
 
@@ -308,20 +302,16 @@ static int __live_heartbeat_off(struct intel_engine_cs *engine)
 
 	intel_engine_pm_get(engine);
 
+	/* Kick once, so that we change an active heartbeat */
 	engine->serial++;
-	flush_delayed_work(&engine->heartbeat.work);
-	if (!delayed_work_pending(&engine->heartbeat.work)) {
-		pr_err("%s: heartbeat not running\n",
-		       engine->name);
-		err = -EINVAL;
-		goto err_pm;
-	}
+	intel_engine_unpark_heartbeat(engine);
 
 	err = intel_engine_set_heartbeat(engine, 0);
 	if (err)
 		goto err_pm;
 
-	engine->serial++;
+	/* The next heartbeat work should cancel the heartbeat */
+	engine->serial++; /* pretend the engine is still active */
 	flush_delayed_work(&engine->heartbeat.work);
 	if (delayed_work_pending(&engine->heartbeat.work)) {
 		pr_err("%s: heartbeat still running\n",
diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c
index 7bab147e4421..4f7298855278 100644
--- a/drivers/gpu/drm/i915/gt/selftest_execlists.c
+++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c
@@ -2324,13 +2324,16 @@ static int __cancel_fail(struct live_preempt_cancel *arg)
 	del_timer_sync(&engine->execlists.preempt);
 	intel_engine_flush_submission(engine);
 
+	engine->props.preempt_timeout_ms = 0;
 	cancel_reset_timeout(engine);
 
-	/* after failure, require heartbeats to reset device */
+	/* after failure, require fast heartbeats to reset device */
 	intel_engine_set_heartbeat(engine, 1);
 	err = wait_for_reset(engine, rq, HZ / 2);
 	intel_engine_set_heartbeat(engine,
 				   engine->defaults.heartbeat_interval_ms);
+
+	engine->props.preempt_timeout_ms = engine->defaults.preempt_timeout_ms;
 	if (err) {
 		pr_err("Cancelled inflight0 request did not reset\n");
 		goto out;
-- 
2.20.1

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply related	[flat|nested] 21+ messages in thread

* [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (2 preceding siblings ...)
  2021-02-05 14:22 ` [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
@ 2021-02-05 14:32 ` Chris Wilson
  2021-02-10 22:38   ` Tang, CQ
  2021-02-05 20:23 ` [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev7) Patchwork
                   ` (11 subsequent siblings)
  15 siblings, 1 reply; 21+ messages in thread
From: Chris Wilson @ 2021-02-05 14:32 UTC (permalink / raw)
  To: intel-gfx; +Cc: Chris Wilson

The heartbeat runs through a few phases that we expect to complete
within a certain number of heartbeat intervals. First we must submit the
heartbeat to the queue, and if the queue is occupied it may take a
couple of intervals before the heartbeat preempts the workload and is
submitted to HW. Once running on HW, completion is not instantaneous as
it may have to first reset the current workload before it itself runs
through the empty request and signals completion. As such, we know that
the heartbeat must take at least the preempt reset timeout and before we
have had a chance to reset the engine, we do not want to issue a global
reset ourselves (simply so that we only try to do one reset at a time
and not confuse ourselves by resetting twice and hitting an innocent.)

So by taking into consideration that once running the request must take
a finite amount of time, we can delay the final completion check to
accommodate that and avoid checking too early (before we've had a chance
to handle any engine resets required).

v2: Attach a callback to flush the work immediately upon the heartbeat
completion and insert the delay before the next.

Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/2853
Suggested-by: CQ Tang <cq.tang@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
 .../gpu/drm/i915/gt/intel_engine_heartbeat.c  | 93 +++++++++++++++++--
 drivers/gpu/drm/i915/gt/intel_engine_types.h  |  1 +
 .../drm/i915/gt/selftest_engine_heartbeat.c   | 65 ++++++-------
 drivers/gpu/drm/i915/gt/selftest_execlists.c  |  5 +-
 4 files changed, 119 insertions(+), 45 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
index 93741a65924a..01d8a04f77b6 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
@@ -20,6 +20,18 @@
  * issue a reset -- in the hope that restores progress.
  */
 
+#define HEARTBEAT_COMPLETION 50u /* milliseconds */
+
+static long completion_timeout(const struct intel_engine_cs *engine)
+{
+	long timeout = HEARTBEAT_COMPLETION;
+
+	if (intel_engine_has_preempt_reset(engine))
+		timeout += READ_ONCE(engine->props.preempt_timeout_ms);
+
+	return msecs_to_jiffies(timeout);
+}
+
 static bool next_heartbeat(struct intel_engine_cs *engine)
 {
 	long delay;
@@ -29,6 +41,26 @@ static bool next_heartbeat(struct intel_engine_cs *engine)
 		return false;
 
 	delay = msecs_to_jiffies_timeout(delay);
+
+	/*
+	 * Once we submit a heartbeat to the HW, we know that it will take
+	 * at least a certain amount of time to complete. On a hanging system
+	 * it will first have to wait for the preempt reset timeout, and
+	 * then it will take some time for the reset to resume with the
+	 * heartbeat and for it to complete. So once we have submitted the
+	 * heartbeat to HW, we can wait a while longer before declaring the
+	 * engine stuck and forcing a reset ourselves. If we do a reset
+	 * and the engine is also doing a reset, it is possible that we
+	 * reset the engine twice, harming an innocent.
+	 *
+	 * Before we have sumitted the heartbeat, we do not want to change
+	 * the interval as we to promote the heartbeat and trigger preemption
+	 * in a deterministic time frame.
+	 */
+	if (engine->heartbeat.systole &&
+	    i915_request_is_active(engine->heartbeat.systole))
+		delay = max(delay, completion_timeout(engine));
+
 	if (delay >= HZ)
 		delay = round_jiffies_up_relative(delay);
 	mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, delay + 1);
@@ -48,12 +80,51 @@ heartbeat_create(struct intel_context *ce, gfp_t gfp)
 	return rq;
 }
 
+static void defibrillator(struct dma_fence *f, struct dma_fence_cb *cb)
+{
+	struct intel_engine_cs *engine =
+		container_of(cb, typeof(*engine), heartbeat.cb);
+
+	if (READ_ONCE(engine->heartbeat.systole))
+		mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, 0);
+}
+
+static void
+untrack_heartbeat(struct intel_engine_cs *engine)
+{
+	struct i915_request *rq;
+
+	rq = fetch_and_zero(&engine->heartbeat.systole);
+	if (!rq)
+		return;
+
+	ENGINE_TRACE(engine, "heartbeat completed: %llx:%lld\n",
+		     rq->fence.context, rq->fence.seqno);
+
+	dma_fence_remove_callback(&rq->fence, &engine->heartbeat.cb);
+	i915_request_put(rq);
+}
+
+static void
+track_heartbeat(struct intel_engine_cs *engine, struct i915_request *rq)
+{
+	ENGINE_TRACE(engine, "heartbeat started: %llx:%lld\n",
+		     rq->fence.context, rq->fence.seqno);
+
+	dma_fence_add_callback(&rq->fence,
+			       &engine->heartbeat.cb,
+			       defibrillator);
+	engine->heartbeat.systole = i915_request_get(rq);
+	if (!next_heartbeat(engine))
+		untrack_heartbeat(engine);
+}
+
 static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq)
 {
 	engine->wakeref_serial = READ_ONCE(engine->serial) + 1;
 	i915_request_add_active_barriers(rq);
 	if (!engine->heartbeat.systole && intel_engine_has_heartbeat(engine))
-		engine->heartbeat.systole = i915_request_get(rq);
+		track_heartbeat(engine, rq);
 }
 
 static void heartbeat_commit(struct i915_request *rq,
@@ -91,10 +162,8 @@ static void heartbeat(struct work_struct *wrk)
 	intel_engine_flush_submission(engine);
 
 	rq = engine->heartbeat.systole;
-	if (rq && i915_request_completed(rq)) {
-		i915_request_put(rq);
-		engine->heartbeat.systole = NULL;
-	}
+	if (rq && i915_request_completed(rq))
+		untrack_heartbeat(engine);
 
 	if (!intel_engine_pm_get_if_awake(engine))
 		return;
@@ -151,6 +220,11 @@ static void heartbeat(struct work_struct *wrk)
 		goto out;
 	}
 
+	/* Just completed one heartbeat, wait a tick before the next */
+	if (rq)
+		goto out;
+
+	/* The engine is parking. We can rest until the next user */
 	serial = READ_ONCE(engine->serial);
 	if (engine->wakeref_serial == serial)
 		goto out;
@@ -174,8 +248,9 @@ static void heartbeat(struct work_struct *wrk)
 unlock:
 	mutex_unlock(&ce->timeline->mutex);
 out:
+	intel_engine_flush_submission(engine);
 	if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+		untrack_heartbeat(engine);
 	intel_engine_pm_put(engine);
 }
 
@@ -189,8 +264,10 @@ void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine)
 
 void intel_engine_park_heartbeat(struct intel_engine_cs *engine)
 {
-	if (cancel_delayed_work(&engine->heartbeat.work))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+	/* completion may rearm work */
+	while (cancel_delayed_work(&engine->heartbeat.work))
+		;
+	untrack_heartbeat(engine);
 }
 
 void intel_engine_init_heartbeat(struct intel_engine_cs *engine)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 7159f9575e65..4956594c8b93 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -347,6 +347,7 @@ struct intel_engine_cs {
 	struct {
 		struct delayed_work work;
 		struct i915_request *systole;
+		struct dma_fence_cb cb;
 		unsigned long blocked;
 	} heartbeat;
 
diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
index b2c369317bf1..812c4a168b01 100644
--- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
@@ -202,47 +202,44 @@ static int cmp_u32(const void *_a, const void *_b)
 
 static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 {
-	const unsigned int error_threshold = max(20000u, jiffies_to_usecs(6));
-	struct intel_context *ce;
-	struct i915_request *rq;
-	ktime_t t0, t1;
+	const unsigned int error_threshold =
+		max(3 * HEARTBEAT_COMPLETION * 1000, jiffies_to_usecs(6));
+	struct intel_context *ce = engine->kernel_context;
 	u32 times[5];
 	int err;
 	int i;
 
-	ce = intel_context_create(engine);
-	if (IS_ERR(ce))
-		return PTR_ERR(ce);
-
 	intel_engine_pm_get(engine);
 
 	err = intel_engine_set_heartbeat(engine, 1);
 	if (err)
 		goto err_pm;
 
+	flush_delayed_work(&engine->heartbeat.work);
+	while (engine->heartbeat.systole)
+		intel_engine_park_heartbeat(engine);
+
 	for (i = 0; i < ARRAY_SIZE(times); i++) {
-		do {
-			/* Manufacture a tick */
-			intel_engine_park_heartbeat(engine);
-			GEM_BUG_ON(engine->heartbeat.systole);
-			engine->serial++; /*  pretend we are not idle! */
-			intel_engine_unpark_heartbeat(engine);
+		struct i915_sched_attr attr = { .priority = I915_PRIORITY_MIN };
+		struct i915_request *rq;
+		ktime_t t0, t1;
 
-			flush_delayed_work(&engine->heartbeat.work);
-			if (!delayed_work_pending(&engine->heartbeat.work)) {
-				pr_err("%s: heartbeat %d did not start\n",
-				       engine->name, i);
-				err = -EINVAL;
-				goto err_pm;
-			}
+		GEM_BUG_ON(READ_ONCE(engine->heartbeat.systole));
 
-			rcu_read_lock();
-			rq = READ_ONCE(engine->heartbeat.systole);
-			if (rq)
-				rq = i915_request_get_rcu(rq);
-			rcu_read_unlock();
-		} while (!rq);
+		/* Manufacture a tick */
+		mutex_lock(&ce->timeline->mutex);
+		rq = heartbeat_create(ce, GFP_KERNEL);
+		if (!IS_ERR(rq)) {
+			i915_request_get(rq);
+			heartbeat_commit(rq, &attr);
+		}
+		mutex_unlock(&ce->timeline->mutex);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto err_reset;
+		}
 
+		/* Time how long before the heartbeat monitor checks */
 		t0 = ktime_get();
 		while (rq == READ_ONCE(engine->heartbeat.systole))
 			yield(); /* work is on the local cpu! */
@@ -275,10 +272,10 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 		err = -EINVAL;
 	}
 
+err_reset:
 	reset_heartbeat(engine);
 err_pm:
 	intel_engine_pm_put(engine);
-	intel_context_put(ce);
 	return err;
 }
 
@@ -308,20 +305,16 @@ static int __live_heartbeat_off(struct intel_engine_cs *engine)
 
 	intel_engine_pm_get(engine);
 
+	/* Kick once, so that we change an active heartbeat */
 	engine->serial++;
-	flush_delayed_work(&engine->heartbeat.work);
-	if (!delayed_work_pending(&engine->heartbeat.work)) {
-		pr_err("%s: heartbeat not running\n",
-		       engine->name);
-		err = -EINVAL;
-		goto err_pm;
-	}
+	intel_engine_unpark_heartbeat(engine);
 
 	err = intel_engine_set_heartbeat(engine, 0);
 	if (err)
 		goto err_pm;
 
-	engine->serial++;
+	/* The next heartbeat work should cancel the heartbeat */
+	engine->serial++; /* pretend the engine is still active */
 	flush_delayed_work(&engine->heartbeat.work);
 	if (delayed_work_pending(&engine->heartbeat.work)) {
 		pr_err("%s: heartbeat still running\n",
diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c
index 7bab147e4421..4f7298855278 100644
--- a/drivers/gpu/drm/i915/gt/selftest_execlists.c
+++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c
@@ -2324,13 +2324,16 @@ static int __cancel_fail(struct live_preempt_cancel *arg)
 	del_timer_sync(&engine->execlists.preempt);
 	intel_engine_flush_submission(engine);
 
+	engine->props.preempt_timeout_ms = 0;
 	cancel_reset_timeout(engine);
 
-	/* after failure, require heartbeats to reset device */
+	/* after failure, require fast heartbeats to reset device */
 	intel_engine_set_heartbeat(engine, 1);
 	err = wait_for_reset(engine, rq, HZ / 2);
 	intel_engine_set_heartbeat(engine,
 				   engine->defaults.heartbeat_interval_ms);
+
+	engine->props.preempt_timeout_ms = engine->defaults.preempt_timeout_ms;
 	if (err) {
 		pr_err("Cancelled inflight0 request did not reset\n");
 		goto out;
-- 
2.20.1

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply related	[flat|nested] 21+ messages in thread

* [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev7)
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (3 preceding siblings ...)
  2021-02-05 14:32 ` Chris Wilson
@ 2021-02-05 20:23 ` Patchwork
  2021-02-06  6:45 ` [Intel-gfx] ✓ Fi.CI.IGT: " Patchwork
                   ` (10 subsequent siblings)
  15 siblings, 0 replies; 21+ messages in thread
From: Patchwork @ 2021-02-05 20:23 UTC (permalink / raw)
  To: Chris Wilson; +Cc: intel-gfx


[-- Attachment #1.1: Type: text/plain, Size: 3210 bytes --]

== Series Details ==

Series: drm/i915/gt: Ratelimit heartbeat completion probing (rev7)
URL   : https://patchwork.freedesktop.org/series/86665/
State : success

== Summary ==

CI Bug Log - changes from CI_DRM_9740 -> Patchwork_19607
====================================================

Summary
-------

  **SUCCESS**

  No regressions found.

  External URL: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/index.html

Known issues
------------

  Here are the changes found in Patchwork_19607 that come from known issues:

### IGT changes ###

#### Issues hit ####

  * igt@gem_exec_suspend@basic-s3:
    - fi-tgl-y:           [PASS][1] -> [DMESG-WARN][2] ([i915#2411] / [i915#402])
   [1]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/fi-tgl-y/igt@gem_exec_suspend@basic-s3.html
   [2]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/fi-tgl-y/igt@gem_exec_suspend@basic-s3.html

  * igt@gem_huc_copy@huc-copy:
    - fi-byt-j1900:       NOTRUN -> [SKIP][3] ([fdo#109271]) +27 similar issues
   [3]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/fi-byt-j1900/igt@gem_huc_copy@huc-copy.html

  * igt@kms_chamelium@hdmi-crc-fast:
    - fi-byt-j1900:       NOTRUN -> [SKIP][4] ([fdo#109271] / [fdo#111827]) +8 similar issues
   [4]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/fi-byt-j1900/igt@kms_chamelium@hdmi-crc-fast.html

  * igt@prime_vgem@basic-fence-flip:
    - fi-tgl-y:           [PASS][5] -> [DMESG-WARN][6] ([i915#402]) +2 similar issues
   [5]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/fi-tgl-y/igt@prime_vgem@basic-fence-flip.html
   [6]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/fi-tgl-y/igt@prime_vgem@basic-fence-flip.html

  
#### Possible fixes ####

  * igt@prime_self_import@basic-with_one_bo_two_files:
    - fi-tgl-y:           [DMESG-WARN][7] ([i915#402]) -> [PASS][8] +1 similar issue
   [7]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/fi-tgl-y/igt@prime_self_import@basic-with_one_bo_two_files.html
   [8]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/fi-tgl-y/igt@prime_self_import@basic-with_one_bo_two_files.html

  
  [fdo#109271]: https://bugs.freedesktop.org/show_bug.cgi?id=109271
  [fdo#111827]: https://bugs.freedesktop.org/show_bug.cgi?id=111827
  [i915#2411]: https://gitlab.freedesktop.org/drm/intel/issues/2411
  [i915#402]: https://gitlab.freedesktop.org/drm/intel/issues/402


Participating hosts (43 -> 38)
------------------------------

  Additional (1): fi-byt-j1900 
  Missing    (6): fi-jsl-1 fi-ilk-m540 fi-hsw-4200u fi-bsw-cyan fi-ctg-p8600 fi-bdw-samus 


Build changes
-------------

  * Linux: CI_DRM_9740 -> Patchwork_19607

  CI-20190529: 20190529
  CI_DRM_9740: d0d6b3dabc3c5f35990abedf7361eb27f7123f4d @ git://anongit.freedesktop.org/gfx-ci/linux
  IGT_5993: b1225ec25d5671a985c5bb48739111d2e8a723cf @ git://anongit.freedesktop.org/xorg/app/intel-gpu-tools
  Patchwork_19607: 4f05cc2083694c58b54d71fd79bed6ef83180f05 @ git://anongit.freedesktop.org/gfx-ci/linux


== Linux commits ==

4f05cc208369 drm/i915/gt: Ratelimit heartbeat completion probing

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/index.html

[-- Attachment #1.2: Type: text/html, Size: 4122 bytes --]

[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply	[flat|nested] 21+ messages in thread

* [Intel-gfx] ✓ Fi.CI.IGT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev7)
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (4 preceding siblings ...)
  2021-02-05 20:23 ` [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev7) Patchwork
@ 2021-02-06  6:45 ` Patchwork
  2021-02-11 12:24 ` [Intel-gfx] [PATCH v3] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (9 subsequent siblings)
  15 siblings, 0 replies; 21+ messages in thread
From: Patchwork @ 2021-02-06  6:45 UTC (permalink / raw)
  To: Chris Wilson; +Cc: intel-gfx


[-- Attachment #1.1: Type: text/plain, Size: 30281 bytes --]

== Series Details ==

Series: drm/i915/gt: Ratelimit heartbeat completion probing (rev7)
URL   : https://patchwork.freedesktop.org/series/86665/
State : success

== Summary ==

CI Bug Log - changes from CI_DRM_9740_full -> Patchwork_19607_full
====================================================

Summary
-------

  **SUCCESS**

  No regressions found.

  

Known issues
------------

  Here are the changes found in Patchwork_19607_full that come from known issues:

### IGT changes ###

#### Issues hit ####

  * igt@gem_create@create-massive:
    - shard-kbl:          NOTRUN -> [DMESG-WARN][1] ([i915#3002])
   [1]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl4/igt@gem_create@create-massive.html

  * igt@gem_exec_fair@basic-none-solo@rcs0:
    - shard-glk:          [PASS][2] -> [FAIL][3] ([i915#2842])
   [2]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-glk6/igt@gem_exec_fair@basic-none-solo@rcs0.html
   [3]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-glk6/igt@gem_exec_fair@basic-none-solo@rcs0.html

  * igt@gem_exec_fair@basic-none@vcs1:
    - shard-kbl:          [PASS][4] -> [FAIL][5] ([i915#2842])
   [4]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-kbl3/igt@gem_exec_fair@basic-none@vcs1.html
   [5]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl4/igt@gem_exec_fair@basic-none@vcs1.html

  * igt@gem_exec_fair@basic-pace-solo@rcs0:
    - shard-tglb:         [PASS][6] -> [FAIL][7] ([i915#2842]) +1 similar issue
   [6]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-tglb7/igt@gem_exec_fair@basic-pace-solo@rcs0.html
   [7]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-tglb7/igt@gem_exec_fair@basic-pace-solo@rcs0.html

  * igt@gem_exec_fair@basic-pace@vcs1:
    - shard-iclb:         NOTRUN -> [FAIL][8] ([i915#2842])
   [8]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb1/igt@gem_exec_fair@basic-pace@vcs1.html

  * igt@gem_exec_schedule@u-fairslice@rcs0:
    - shard-skl:          NOTRUN -> [DMESG-WARN][9] ([i915#1610] / [i915#2803])
   [9]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl5/igt@gem_exec_schedule@u-fairslice@rcs0.html

  * igt@gem_exec_whisper@basic-contexts-forked-all:
    - shard-glk:          [PASS][10] -> [DMESG-WARN][11] ([i915#118] / [i915#95])
   [10]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-glk2/igt@gem_exec_whisper@basic-contexts-forked-all.html
   [11]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-glk8/igt@gem_exec_whisper@basic-contexts-forked-all.html

  * igt@gem_render_copy@y-tiled-to-vebox-y-tiled:
    - shard-iclb:         NOTRUN -> [SKIP][12] ([i915#768]) +1 similar issue
   [12]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb3/igt@gem_render_copy@y-tiled-to-vebox-y-tiled.html

  * igt@gem_userptr_blits@vma-merge:
    - shard-apl:          NOTRUN -> [INCOMPLETE][13] ([i915#2502] / [i915#2667])
   [13]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-apl1/igt@gem_userptr_blits@vma-merge.html
    - shard-kbl:          NOTRUN -> [INCOMPLETE][14] ([i915#2502] / [i915#2667])
   [14]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl6/igt@gem_userptr_blits@vma-merge.html

  * igt@gem_vm_create@destroy-race:
    - shard-tglb:         [PASS][15] -> [TIMEOUT][16] ([i915#2795])
   [15]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-tglb2/igt@gem_vm_create@destroy-race.html
   [16]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-tglb7/igt@gem_vm_create@destroy-race.html

  * igt@gen3_render_tiledy_blits:
    - shard-iclb:         NOTRUN -> [SKIP][17] ([fdo#109289])
   [17]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb5/igt@gen3_render_tiledy_blits.html

  * igt@gen9_exec_parse@allowed-all:
    - shard-glk:          [PASS][18] -> [DMESG-WARN][19] ([i915#1436] / [i915#716])
   [18]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-glk1/igt@gen9_exec_parse@allowed-all.html
   [19]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-glk3/igt@gen9_exec_parse@allowed-all.html

  * igt@gen9_exec_parse@valid-registers:
    - shard-iclb:         NOTRUN -> [SKIP][20] ([fdo#112306])
   [20]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb5/igt@gen9_exec_parse@valid-registers.html

  * igt@i915_pm_rc6_residency@rc6-fence:
    - shard-iclb:         NOTRUN -> [WARN][21] ([i915#1804] / [i915#2684])
   [21]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb3/igt@i915_pm_rc6_residency@rc6-fence.html

  * igt@i915_pm_rpm@modeset-lpsp-stress:
    - shard-apl:          NOTRUN -> [SKIP][22] ([fdo#109271]) +33 similar issues
   [22]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-apl4/igt@i915_pm_rpm@modeset-lpsp-stress.html

  * igt@kms_big_fb@x-tiled-64bpp-rotate-270:
    - shard-iclb:         NOTRUN -> [SKIP][23] ([fdo#110725] / [fdo#111614]) +1 similar issue
   [23]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb5/igt@kms_big_fb@x-tiled-64bpp-rotate-270.html

  * igt@kms_big_joiner@invalid-modeset:
    - shard-skl:          NOTRUN -> [SKIP][24] ([fdo#109271] / [i915#2705])
   [24]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl5/igt@kms_big_joiner@invalid-modeset.html

  * igt@kms_chamelium@hdmi-hpd-storm-disable:
    - shard-skl:          NOTRUN -> [SKIP][25] ([fdo#109271] / [fdo#111827]) +4 similar issues
   [25]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl4/igt@kms_chamelium@hdmi-hpd-storm-disable.html

  * igt@kms_chamelium@vga-hpd-without-ddc:
    - shard-kbl:          NOTRUN -> [SKIP][26] ([fdo#109271] / [fdo#111827]) +4 similar issues
   [26]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl3/igt@kms_chamelium@vga-hpd-without-ddc.html

  * igt@kms_color@pipe-c-ctm-0-75:
    - shard-iclb:         NOTRUN -> [FAIL][27] ([i915#1149] / [i915#315])
   [27]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb3/igt@kms_color@pipe-c-ctm-0-75.html

  * igt@kms_color@pipe-d-ctm-0-5:
    - shard-skl:          NOTRUN -> [SKIP][28] ([fdo#109271]) +73 similar issues
   [28]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl9/igt@kms_color@pipe-d-ctm-0-5.html

  * igt@kms_color@pipe-d-ctm-red-to-blue:
    - shard-iclb:         NOTRUN -> [SKIP][29] ([fdo#109278] / [i915#1149])
   [29]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb5/igt@kms_color@pipe-d-ctm-red-to-blue.html

  * igt@kms_color_chamelium@pipe-c-ctm-green-to-red:
    - shard-iclb:         NOTRUN -> [SKIP][30] ([fdo#109284] / [fdo#111827])
   [30]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb5/igt@kms_color_chamelium@pipe-c-ctm-green-to-red.html

  * igt@kms_color_chamelium@pipe-d-ctm-0-25:
    - shard-apl:          NOTRUN -> [SKIP][31] ([fdo#109271] / [fdo#111827]) +4 similar issues
   [31]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-apl1/igt@kms_color_chamelium@pipe-d-ctm-0-25.html

  * igt@kms_content_protection@legacy:
    - shard-kbl:          NOTRUN -> [TIMEOUT][32] ([i915#1319])
   [32]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl3/igt@kms_content_protection@legacy.html

  * igt@kms_content_protection@uevent:
    - shard-kbl:          NOTRUN -> [FAIL][33] ([i915#2105])
   [33]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl1/igt@kms_content_protection@uevent.html

  * igt@kms_cursor_crc@pipe-a-cursor-256x85-offscreen:
    - shard-skl:          [PASS][34] -> [FAIL][35] ([i915#54]) +5 similar issues
   [34]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl10/igt@kms_cursor_crc@pipe-a-cursor-256x85-offscreen.html
   [35]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl2/igt@kms_cursor_crc@pipe-a-cursor-256x85-offscreen.html

  * igt@kms_cursor_crc@pipe-a-cursor-512x170-sliding:
    - shard-kbl:          NOTRUN -> [SKIP][36] ([fdo#109271]) +47 similar issues
   [36]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl3/igt@kms_cursor_crc@pipe-a-cursor-512x170-sliding.html

  * igt@kms_cursor_crc@pipe-a-cursor-suspend:
    - shard-kbl:          [PASS][37] -> [DMESG-WARN][38] ([i915#180]) +5 similar issues
   [37]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-kbl1/igt@kms_cursor_crc@pipe-a-cursor-suspend.html
   [38]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl2/igt@kms_cursor_crc@pipe-a-cursor-suspend.html

  * igt@kms_cursor_crc@pipe-c-cursor-128x128-random:
    - shard-skl:          NOTRUN -> [FAIL][39] ([i915#54]) +1 similar issue
   [39]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl5/igt@kms_cursor_crc@pipe-c-cursor-128x128-random.html

  * igt@kms_cursor_legacy@cursora-vs-flipa-varying-size:
    - shard-skl:          [PASS][40] -> [DMESG-WARN][41] ([i915#1982])
   [40]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl6/igt@kms_cursor_legacy@cursora-vs-flipa-varying-size.html
   [41]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl7/igt@kms_cursor_legacy@cursora-vs-flipa-varying-size.html

  * igt@kms_flip@2x-flip-vs-dpms-off-vs-modeset:
    - shard-iclb:         NOTRUN -> [SKIP][42] ([fdo#109274])
   [42]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb3/igt@kms_flip@2x-flip-vs-dpms-off-vs-modeset.html

  * igt@kms_flip@plain-flip-fb-recreate-interruptible@a-edp1:
    - shard-skl:          NOTRUN -> [FAIL][43] ([i915#2122])
   [43]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl4/igt@kms_flip@plain-flip-fb-recreate-interruptible@a-edp1.html

  * igt@kms_flip@plain-flip-fb-recreate@b-edp1:
    - shard-skl:          [PASS][44] -> [FAIL][45] ([i915#2122]) +2 similar issues
   [44]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl4/igt@kms_flip@plain-flip-fb-recreate@b-edp1.html
   [45]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl1/igt@kms_flip@plain-flip-fb-recreate@b-edp1.html

  * igt@kms_flip_scaled_crc@flip-32bpp-ytile-to-32bpp-ytilegen12rcccs:
    - shard-skl:          NOTRUN -> [SKIP][46] ([fdo#109271] / [i915#2672])
   [46]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl4/igt@kms_flip_scaled_crc@flip-32bpp-ytile-to-32bpp-ytilegen12rcccs.html

  * igt@kms_frontbuffer_tracking@fbc-2p-scndscrn-pri-indfb-draw-pwrite:
    - shard-iclb:         NOTRUN -> [SKIP][47] ([fdo#109280]) +3 similar issues
   [47]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb5/igt@kms_frontbuffer_tracking@fbc-2p-scndscrn-pri-indfb-draw-pwrite.html

  * igt@kms_pipe_crc_basic@hang-read-crc-pipe-d:
    - shard-skl:          NOTRUN -> [SKIP][48] ([fdo#109271] / [i915#533]) +1 similar issue
   [48]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl9/igt@kms_pipe_crc_basic@hang-read-crc-pipe-d.html

  * igt@kms_pipe_crc_basic@nonblocking-crc-pipe-d-frame-sequence:
    - shard-kbl:          NOTRUN -> [SKIP][49] ([fdo#109271] / [i915#533])
   [49]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl1/igt@kms_pipe_crc_basic@nonblocking-crc-pipe-d-frame-sequence.html

  * igt@kms_pipe_crc_basic@read-crc-pipe-d-frame-sequence:
    - shard-iclb:         NOTRUN -> [SKIP][50] ([fdo#109278]) +5 similar issues
   [50]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb5/igt@kms_pipe_crc_basic@read-crc-pipe-d-frame-sequence.html

  * igt@kms_pipe_crc_basic@suspend-read-crc-pipe-c:
    - shard-apl:          [PASS][51] -> [DMESG-WARN][52] ([i915#180])
   [51]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-apl8/igt@kms_pipe_crc_basic@suspend-read-crc-pipe-c.html
   [52]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-apl3/igt@kms_pipe_crc_basic@suspend-read-crc-pipe-c.html

  * igt@kms_plane@plane-panning-bottom-right-suspend-pipe-c-planes:
    - shard-skl:          [PASS][53] -> [INCOMPLETE][54] ([i915#198])
   [53]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl3/igt@kms_plane@plane-panning-bottom-right-suspend-pipe-c-planes.html
   [54]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl1/igt@kms_plane@plane-panning-bottom-right-suspend-pipe-c-planes.html

  * igt@kms_plane@plane-position-hole-pipe-c-planes:
    - shard-skl:          [PASS][55] -> [FAIL][56] ([i915#2472])
   [55]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl4/igt@kms_plane@plane-position-hole-pipe-c-planes.html
   [56]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl6/igt@kms_plane@plane-position-hole-pipe-c-planes.html

  * igt@kms_plane_alpha_blend@pipe-a-constant-alpha-max:
    - shard-skl:          NOTRUN -> [FAIL][57] ([fdo#108145] / [i915#265])
   [57]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl4/igt@kms_plane_alpha_blend@pipe-a-constant-alpha-max.html

  * igt@kms_plane_alpha_blend@pipe-c-alpha-7efc:
    - shard-apl:          NOTRUN -> [FAIL][58] ([fdo#108145] / [i915#265])
   [58]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-apl1/igt@kms_plane_alpha_blend@pipe-c-alpha-7efc.html

  * igt@kms_plane_alpha_blend@pipe-c-constant-alpha-max:
    - shard-kbl:          NOTRUN -> [FAIL][59] ([fdo#108145] / [i915#265])
   [59]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl3/igt@kms_plane_alpha_blend@pipe-c-constant-alpha-max.html

  * igt@kms_plane_alpha_blend@pipe-c-coverage-7efc:
    - shard-skl:          [PASS][60] -> [FAIL][61] ([fdo#108145] / [i915#265]) +1 similar issue
   [60]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl9/igt@kms_plane_alpha_blend@pipe-c-coverage-7efc.html
   [61]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl6/igt@kms_plane_alpha_blend@pipe-c-coverage-7efc.html

  * igt@kms_psr2_sf@overlay-plane-update-sf-dmg-area-5:
    - shard-iclb:         NOTRUN -> [SKIP][62] ([i915#658])
   [62]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb3/igt@kms_psr2_sf@overlay-plane-update-sf-dmg-area-5.html

  * igt@kms_psr2_sf@overlay-primary-update-sf-dmg-area-5:
    - shard-skl:          NOTRUN -> [SKIP][63] ([fdo#109271] / [i915#658])
   [63]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl9/igt@kms_psr2_sf@overlay-primary-update-sf-dmg-area-5.html

  * igt@kms_psr@psr2_no_drrs:
    - shard-iclb:         [PASS][64] -> [SKIP][65] ([fdo#109441]) +1 similar issue
   [64]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-iclb2/igt@kms_psr@psr2_no_drrs.html
   [65]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb3/igt@kms_psr@psr2_no_drrs.html

  * igt@kms_vblank@pipe-d-wait-idle:
    - shard-apl:          NOTRUN -> [SKIP][66] ([fdo#109271] / [i915#533]) +1 similar issue
   [66]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-apl4/igt@kms_vblank@pipe-d-wait-idle.html

  * igt@kms_writeback@writeback-fb-id:
    - shard-apl:          NOTRUN -> [SKIP][67] ([fdo#109271] / [i915#2437]) +1 similar issue
   [67]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-apl1/igt@kms_writeback@writeback-fb-id.html

  * igt@perf@polling-small-buf:
    - shard-skl:          NOTRUN -> [FAIL][68] ([i915#1722])
   [68]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl3/igt@perf@polling-small-buf.html

  * igt@prime_nv_pcopy@test3_4:
    - shard-iclb:         NOTRUN -> [SKIP][69] ([fdo#109291])
   [69]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb3/igt@prime_nv_pcopy@test3_4.html

  * igt@sysfs_clients@recycle:
    - shard-snb:          [PASS][70] -> [FAIL][71] ([i915#3028])
   [70]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-snb5/igt@sysfs_clients@recycle.html
   [71]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-snb7/igt@sysfs_clients@recycle.html

  * igt@sysfs_clients@split-10@rcs0:
    - shard-apl:          [PASS][72] -> [SKIP][73] ([fdo#109271] / [i915#3026])
   [72]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-apl2/igt@sysfs_clients@split-10@rcs0.html
   [73]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-apl2/igt@sysfs_clients@split-10@rcs0.html

  
#### Possible fixes ####

  * igt@gem_ctx_isolation@preservation-s3@rcs0:
    - shard-apl:          [DMESG-WARN][74] ([i915#180]) -> [PASS][75] +3 similar issues
   [74]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-apl3/igt@gem_ctx_isolation@preservation-s3@rcs0.html
   [75]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-apl4/igt@gem_ctx_isolation@preservation-s3@rcs0.html

  * igt@gem_ctx_persistence@smoketest:
    - shard-tglb:         [FAIL][76] ([i915#2896]) -> [PASS][77]
   [76]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-tglb2/igt@gem_ctx_persistence@smoketest.html
   [77]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-tglb2/igt@gem_ctx_persistence@smoketest.html

  * igt@gem_eio@in-flight-suspend:
    - shard-kbl:          [DMESG-WARN][78] ([i915#1037] / [i915#180]) -> [PASS][79]
   [78]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-kbl2/igt@gem_eio@in-flight-suspend.html
   [79]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl4/igt@gem_eio@in-flight-suspend.html

  * igt@gem_exec_balancer@hang:
    - shard-iclb:         [INCOMPLETE][80] ([i915#1895] / [i915#2295] / [i915#3031]) -> [PASS][81]
   [80]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-iclb2/igt@gem_exec_balancer@hang.html
   [81]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb3/igt@gem_exec_balancer@hang.html

  * igt@gem_exec_fair@basic-flow@rcs0:
    - shard-tglb:         [FAIL][82] ([i915#2842]) -> [PASS][83] +2 similar issues
   [82]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-tglb1/igt@gem_exec_fair@basic-flow@rcs0.html
   [83]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-tglb8/igt@gem_exec_fair@basic-flow@rcs0.html

  * igt@gem_exec_fair@basic-none-share@rcs0:
    - shard-iclb:         [FAIL][84] ([i915#2842]) -> [PASS][85]
   [84]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-iclb8/igt@gem_exec_fair@basic-none-share@rcs0.html
   [85]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb6/igt@gem_exec_fair@basic-none-share@rcs0.html
    - shard-glk:          [FAIL][86] ([i915#2842]) -> [PASS][87]
   [86]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-glk4/igt@gem_exec_fair@basic-none-share@rcs0.html
   [87]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-glk4/igt@gem_exec_fair@basic-none-share@rcs0.html

  * igt@gem_exec_schedule@u-fairslice@rcs0:
    - shard-iclb:         [DMESG-WARN][88] ([i915#2803]) -> [PASS][89]
   [88]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-iclb1/igt@gem_exec_schedule@u-fairslice@rcs0.html
   [89]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb5/igt@gem_exec_schedule@u-fairslice@rcs0.html

  * igt@gem_exec_whisper@basic-queues-priority:
    - shard-glk:          [DMESG-WARN][90] ([i915#118] / [i915#95]) -> [PASS][91]
   [90]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-glk3/igt@gem_exec_whisper@basic-queues-priority.html
   [91]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-glk7/igt@gem_exec_whisper@basic-queues-priority.html

  * igt@gem_workarounds@suspend-resume-fd:
    - shard-kbl:          [DMESG-WARN][92] ([i915#180]) -> [PASS][93] +2 similar issues
   [92]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-kbl7/igt@gem_workarounds@suspend-resume-fd.html
   [93]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl6/igt@gem_workarounds@suspend-resume-fd.html

  * igt@i915_selftest@live@client:
    - shard-glk:          [DMESG-FAIL][94] ([i915#3047]) -> [PASS][95]
   [94]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-glk9/igt@i915_selftest@live@client.html
   [95]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-glk2/igt@i915_selftest@live@client.html

  * igt@kms_color@pipe-a-ctm-0-75:
    - shard-skl:          [DMESG-WARN][96] ([i915#1982]) -> [PASS][97]
   [96]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl1/igt@kms_color@pipe-a-ctm-0-75.html
   [97]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl2/igt@kms_color@pipe-a-ctm-0-75.html

  * igt@kms_cursor_crc@pipe-b-cursor-256x85-random:
    - shard-skl:          [FAIL][98] ([i915#54]) -> [PASS][99] +5 similar issues
   [98]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl10/igt@kms_cursor_crc@pipe-b-cursor-256x85-random.html
   [99]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl2/igt@kms_cursor_crc@pipe-b-cursor-256x85-random.html

  * igt@kms_cursor_legacy@flip-vs-cursor-busy-crc-atomic:
    - shard-skl:          [FAIL][100] ([i915#2346]) -> [PASS][101]
   [100]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl6/igt@kms_cursor_legacy@flip-vs-cursor-busy-crc-atomic.html
   [101]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl7/igt@kms_cursor_legacy@flip-vs-cursor-busy-crc-atomic.html

  * igt@kms_flip@flip-vs-expired-vblank-interruptible@a-edp1:
    - shard-tglb:         [FAIL][102] ([i915#2598]) -> [PASS][103]
   [102]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-tglb1/igt@kms_flip@flip-vs-expired-vblank-interruptible@a-edp1.html
   [103]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-tglb8/igt@kms_flip@flip-vs-expired-vblank-interruptible@a-edp1.html

  * igt@kms_flip@plain-flip-ts-check@b-edp1:
    - shard-skl:          [FAIL][104] ([i915#2122]) -> [PASS][105]
   [104]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl1/igt@kms_flip@plain-flip-ts-check@b-edp1.html
   [105]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl2/igt@kms_flip@plain-flip-ts-check@b-edp1.html

  * igt@kms_pipe_crc_basic@suspend-read-crc-pipe-a:
    - shard-skl:          [INCOMPLETE][106] ([i915#198] / [i915#1982]) -> [PASS][107]
   [106]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl3/igt@kms_pipe_crc_basic@suspend-read-crc-pipe-a.html
   [107]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl4/igt@kms_pipe_crc_basic@suspend-read-crc-pipe-a.html

  * igt@kms_plane_alpha_blend@pipe-a-constant-alpha-min:
    - shard-skl:          [FAIL][108] ([fdo#108145] / [i915#265]) -> [PASS][109]
   [108]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl1/igt@kms_plane_alpha_blend@pipe-a-constant-alpha-min.html
   [109]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl2/igt@kms_plane_alpha_blend@pipe-a-constant-alpha-min.html

  * igt@kms_psr2_su@page_flip:
    - shard-iclb:         [SKIP][110] ([fdo#109642] / [fdo#111068] / [i915#658]) -> [PASS][111]
   [110]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-iclb3/igt@kms_psr2_su@page_flip.html
   [111]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb2/igt@kms_psr2_su@page_flip.html

  * igt@kms_psr@psr2_sprite_blt:
    - shard-iclb:         [SKIP][112] ([fdo#109441]) -> [PASS][113]
   [112]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-iclb4/igt@kms_psr@psr2_sprite_blt.html
   [113]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb2/igt@kms_psr@psr2_sprite_blt.html

  * igt@perf_pmu@module-unload:
    - shard-skl:          [DMESG-WARN][114] ([i915#1982] / [i915#262]) -> [PASS][115]
   [114]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl1/igt@perf_pmu@module-unload.html
   [115]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl2/igt@perf_pmu@module-unload.html

  * {igt@sysfs_clients@recycle-many}:
    - shard-kbl:          [FAIL][116] ([i915#3028]) -> [PASS][117]
   [116]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-kbl1/igt@sysfs_clients@recycle-many.html
   [117]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl2/igt@sysfs_clients@recycle-many.html

  
#### Warnings ####

  * igt@i915_pm_rc6_residency@rc6-idle:
    - shard-iclb:         [WARN][118] ([i915#2684]) -> [WARN][119] ([i915#2681] / [i915#2684])
   [118]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-iclb2/igt@i915_pm_rc6_residency@rc6-idle.html
   [119]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb1/igt@i915_pm_rc6_residency@rc6-idle.html

  * igt@kms_psr2_sf@overlay-primary-update-sf-dmg-area-5:
    - shard-iclb:         [SKIP][120] ([i915#2920]) -> [SKIP][121] ([i915#658])
   [120]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-iclb2/igt@kms_psr2_sf@overlay-primary-update-sf-dmg-area-5.html
   [121]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb1/igt@kms_psr2_sf@overlay-primary-update-sf-dmg-area-5.html

  * igt@runner@aborted:
    - shard-kbl:          ([FAIL][122], [FAIL][123], [FAIL][124], [FAIL][125], [FAIL][126]) ([i915#1436] / [i915#1814] / [i915#2295] / [i915#3002] / [i915#602]) -> ([FAIL][127], [FAIL][128], [FAIL][129], [FAIL][130], [FAIL][131], [FAIL][132], [FAIL][133]) ([i915#1436] / [i915#1814] / [i915#2295] / [i915#2505] / [i915#3002])
   [122]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-kbl7/igt@runner@aborted.html
   [123]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-kbl2/igt@runner@aborted.html
   [124]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-kbl7/igt@runner@aborted.html
   [125]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-kbl2/igt@runner@aborted.html
   [126]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-kbl7/igt@runner@aborted.html
   [127]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl4/igt@runner@aborted.html
   [128]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl2/igt@runner@aborted.html
   [129]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl6/igt@runner@aborted.html
   [130]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl6/igt@runner@aborted.html
   [131]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl6/igt@runner@aborted.html
   [132]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl7/igt@runner@aborted.html
   [133]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-kbl6/igt@runner@aborted.html
    - shard-iclb:         ([FAIL][134], [FAIL][135], [FAIL][136], [FAIL][137]) ([i915#2295] / [i915#2426] / [i915#2724] / [i915#3002]) -> ([FAIL][138], [FAIL][139], [FAIL][140]) ([i915#2295] / [i915#2724] / [i915#3002])
   [134]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-iclb6/igt@runner@aborted.html
   [135]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-iclb1/igt@runner@aborted.html
   [136]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-iclb4/igt@runner@aborted.html
   [137]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-iclb3/igt@runner@aborted.html
   [138]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb8/igt@runner@aborted.html
   [139]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb2/igt@runner@aborted.html
   [140]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-iclb4/igt@runner@aborted.html
    - shard-skl:          ([FAIL][141], [FAIL][142], [FAIL][143], [FAIL][144]) ([i915#1814] / [i915#2029] / [i915#2295] / [i915#3002]) -> ([FAIL][145], [FAIL][146], [FAIL][147], [FAIL][148]) ([i915#2295] / [i915#2426] / [i915#3002])
   [141]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl2/igt@runner@aborted.html
   [142]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl5/igt@runner@aborted.html
   [143]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl10/igt@runner@aborted.html
   [144]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9740/shard-skl1/igt@runner@aborted.html
   [145]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl1/igt@runner@aborted.html
   [146]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl3/igt@runner@aborted.html
   [147]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl5/igt@runner@aborted.html
   [148]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/shard-skl8/igt@runner@aborted.html

  
  {name}: This element is suppressed. This means it is ignored when computing
          the status of the difference (SUCCESS, WARNING, or FAILURE).

  [fdo#108145]: https://bugs.freedesktop.org/show_bug.cgi?id=108145
  [fdo#109271]: https://bugs.freedesktop.org/show_bug.cgi?id=109271
  [fdo#109274]: https://bugs.freedesktop.org/show_bug.cgi?id=109274
  [fdo#109278]: https://bugs.freedesktop.org/show_bug.cgi?id=109278
  [fdo#109280]: https://bugs.freedesktop.org/show_bug.cgi?id=109280
  [fdo#109284]: https://bugs.freedesktop.org/show_bug.cgi?id=109284
  [fdo#109289]: https://bugs.freedesktop.org/show_bug.cgi?id=109289
  [fdo#109291]: https://bugs.freedesktop.org/show_bug.cgi?id=109291
  [fdo#109441]: https://bugs.freedesktop.org/show_bug.cgi?id=109441
  [fdo#109642]: https://bugs.freedesktop.org/show_bug.cgi?id=109642
  [fdo#110725]: https://bugs.freedesktop.org/show_bug.cgi?id=110725
  [fdo#111068]: https://bugs.freedesktop.org/show_bug.cgi?id=111068
  [fdo#111614]: https://bugs.freedesktop.org/show_bug.cgi?id=111614
  [fdo#111827]: https://bugs.freedesktop.org/show_bug.cgi?id=111827
  [fdo#112306]: https://bugs.freedesktop.org/show_bug.cgi?id=112306
  [i915#1037]: https://gitlab.freedesktop.org/drm/intel/issues/1037
  [i915#1149]: https://gitlab.freedesktop.org/drm/intel/issues/1149
  [i915#118]: https://gitlab.freedesktop.org/drm/intel/issues/118
  [i915#1319]: https://gitlab.freedesktop.org/drm/intel/issues/1319
  [i915#1436]: https://gitlab.freedesktop.org/drm/intel/issues/1436
  [i915#1610]: https://gitlab.freedesktop.org/drm/intel/issues/1610
  [i915#1722]: https://gitlab.freedesktop.org/drm/intel/issues/1722
  [i915#180]: http

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19607/index.html

[-- Attachment #1.2: Type: text/html, Size: 35259 bytes --]

[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply	[flat|nested] 21+ messages in thread

* Re: [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing
  2021-02-05 14:32 ` Chris Wilson
@ 2021-02-10 22:38   ` Tang, CQ
  0 siblings, 0 replies; 21+ messages in thread
From: Tang, CQ @ 2021-02-10 22:38 UTC (permalink / raw)
  To: Chris Wilson, intel-gfx

Chris,
    I applied this patch to DII.  I could not apply automatically, so I do it manually. The only problem is following in selftest code:
Dii code:
+err_reset:
	intel_engine_set_heartbeat(engine, CONFIG_DRM_I915_HEARTBEAT_INTERVAL);
your code:
 +err_reset:
	reset_heartbeat(engine);

Anyway, I run the patch on PVC simics with selftest gt_hearbeat, but I catch following error:
[   64.836061] i915: Running intel_heartbeat_live_selftests/live_heartbeat_fast
[   64.838307] heartbeat bcs0 heartbeat {seqno:1:5, prio:2147483646} not ticking
[   64.840308] heartbeat        Awake? 3
[   64.841262] heartbeat        Barriers?: no
[   64.842311] heartbeat        Latency: 2260us
[   64.843422] heartbeat        Forcewake: 0 domains, 0 active
[   64.844857] heartbeat        Heartbeat: 7 ms ago
[   64.846071] heartbeat        Reset count: -1 (global 0)
......

It detects GPU hang.

What is the problem? Do you have the patch for DII for me to try?

--CQ

> -----Original Message-----
> From: Chris Wilson <chris@chris-wilson.co.uk>
> Sent: Friday, February 5, 2021 6:32 AM
> To: intel-gfx@lists.freedesktop.org
> Cc: Chris Wilson <chris@chris-wilson.co.uk>; Tang, CQ <cq.tang@intel.com>
> Subject: [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing
> 
> The heartbeat runs through a few phases that we expect to complete within
> a certain number of heartbeat intervals. First we must submit the heartbeat
> to the queue, and if the queue is occupied it may take a couple of intervals
> before the heartbeat preempts the workload and is submitted to HW. Once
> running on HW, completion is not instantaneous as it may have to first reset
> the current workload before it itself runs through the empty request and
> signals completion. As such, we know that the heartbeat must take at least
> the preempt reset timeout and before we have had a chance to reset the
> engine, we do not want to issue a global reset ourselves (simply so that we
> only try to do one reset at a time and not confuse ourselves by resetting
> twice and hitting an innocent.)
> 
> So by taking into consideration that once running the request must take a
> finite amount of time, we can delay the final completion check to
> accommodate that and avoid checking too early (before we've had a chance
> to handle any engine resets required).
> 
> v2: Attach a callback to flush the work immediately upon the heartbeat
> completion and insert the delay before the next.
> 
> Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/2853
> Suggested-by: CQ Tang <cq.tang@intel.com>
> Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
> ---
>  .../gpu/drm/i915/gt/intel_engine_heartbeat.c  | 93 +++++++++++++++++--
> drivers/gpu/drm/i915/gt/intel_engine_types.h  |  1 +
>  .../drm/i915/gt/selftest_engine_heartbeat.c   | 65 ++++++-------
>  drivers/gpu/drm/i915/gt/selftest_execlists.c  |  5 +-
>  4 files changed, 119 insertions(+), 45 deletions(-)
> 
> diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
> b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
> index 93741a65924a..01d8a04f77b6 100644
> --- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
> +++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
> @@ -20,6 +20,18 @@
>   * issue a reset -- in the hope that restores progress.
>   */
> 
> +#define HEARTBEAT_COMPLETION 50u /* milliseconds */
> +
> +static long completion_timeout(const struct intel_engine_cs *engine) {
> +	long timeout = HEARTBEAT_COMPLETION;
> +
> +	if (intel_engine_has_preempt_reset(engine))
> +		timeout += READ_ONCE(engine-
> >props.preempt_timeout_ms);
> +
> +	return msecs_to_jiffies(timeout);
> +}
> +
>  static bool next_heartbeat(struct intel_engine_cs *engine)  {
>  	long delay;
> @@ -29,6 +41,26 @@ static bool next_heartbeat(struct intel_engine_cs
> *engine)
>  		return false;
> 
>  	delay = msecs_to_jiffies_timeout(delay);
> +
> +	/*
> +	 * Once we submit a heartbeat to the HW, we know that it will take
> +	 * at least a certain amount of time to complete. On a hanging system
> +	 * it will first have to wait for the preempt reset timeout, and
> +	 * then it will take some time for the reset to resume with the
> +	 * heartbeat and for it to complete. So once we have submitted the
> +	 * heartbeat to HW, we can wait a while longer before declaring the
> +	 * engine stuck and forcing a reset ourselves. If we do a reset
> +	 * and the engine is also doing a reset, it is possible that we
> +	 * reset the engine twice, harming an innocent.
> +	 *
> +	 * Before we have sumitted the heartbeat, we do not want to
> change
> +	 * the interval as we to promote the heartbeat and trigger
> preemption
> +	 * in a deterministic time frame.
> +	 */
> +	if (engine->heartbeat.systole &&
> +	    i915_request_is_active(engine->heartbeat.systole))
> +		delay = max(delay, completion_timeout(engine));
> +
>  	if (delay >= HZ)
>  		delay = round_jiffies_up_relative(delay);
>  	mod_delayed_work(system_highpri_wq, &engine->heartbeat.work,
> delay + 1); @@ -48,12 +80,51 @@ heartbeat_create(struct intel_context *ce,
> gfp_t gfp)
>  	return rq;
>  }
> 
> +static void defibrillator(struct dma_fence *f, struct dma_fence_cb *cb)
> +{
> +	struct intel_engine_cs *engine =
> +		container_of(cb, typeof(*engine), heartbeat.cb);
> +
> +	if (READ_ONCE(engine->heartbeat.systole))
> +		mod_delayed_work(system_highpri_wq, &engine-
> >heartbeat.work, 0); }
> +
> +static void
> +untrack_heartbeat(struct intel_engine_cs *engine) {
> +	struct i915_request *rq;
> +
> +	rq = fetch_and_zero(&engine->heartbeat.systole);
> +	if (!rq)
> +		return;
> +
> +	ENGINE_TRACE(engine, "heartbeat completed: %llx:%lld\n",
> +		     rq->fence.context, rq->fence.seqno);
> +
> +	dma_fence_remove_callback(&rq->fence, &engine->heartbeat.cb);
> +	i915_request_put(rq);
> +}
> +
> +static void
> +track_heartbeat(struct intel_engine_cs *engine, struct i915_request
> +*rq) {
> +	ENGINE_TRACE(engine, "heartbeat started: %llx:%lld\n",
> +		     rq->fence.context, rq->fence.seqno);
> +
> +	dma_fence_add_callback(&rq->fence,
> +			       &engine->heartbeat.cb,
> +			       defibrillator);
> +	engine->heartbeat.systole = i915_request_get(rq);
> +	if (!next_heartbeat(engine))
> +		untrack_heartbeat(engine);
> +}
> +
>  static void idle_pulse(struct intel_engine_cs *engine, struct i915_request
> *rq)  {
>  	engine->wakeref_serial = READ_ONCE(engine->serial) + 1;
>  	i915_request_add_active_barriers(rq);
>  	if (!engine->heartbeat.systole &&
> intel_engine_has_heartbeat(engine))
> -		engine->heartbeat.systole = i915_request_get(rq);
> +		track_heartbeat(engine, rq);
>  }
> 
>  static void heartbeat_commit(struct i915_request *rq, @@ -91,10 +162,8
> @@ static void heartbeat(struct work_struct *wrk)
>  	intel_engine_flush_submission(engine);
> 
>  	rq = engine->heartbeat.systole;
> -	if (rq && i915_request_completed(rq)) {
> -		i915_request_put(rq);
> -		engine->heartbeat.systole = NULL;
> -	}
> +	if (rq && i915_request_completed(rq))
> +		untrack_heartbeat(engine);
> 
>  	if (!intel_engine_pm_get_if_awake(engine))
>  		return;
> @@ -151,6 +220,11 @@ static void heartbeat(struct work_struct *wrk)
>  		goto out;
>  	}
> 
> +	/* Just completed one heartbeat, wait a tick before the next */
> +	if (rq)
> +		goto out;
> +
> +	/* The engine is parking. We can rest until the next user */
>  	serial = READ_ONCE(engine->serial);
>  	if (engine->wakeref_serial == serial)
>  		goto out;
> @@ -174,8 +248,9 @@ static void heartbeat(struct work_struct *wrk)
>  unlock:
>  	mutex_unlock(&ce->timeline->mutex);
>  out:
> +	intel_engine_flush_submission(engine);
>  	if (!engine->i915->params.enable_hangcheck
> || !next_heartbeat(engine))
> -		i915_request_put(fetch_and_zero(&engine-
> >heartbeat.systole));
> +		untrack_heartbeat(engine);
>  	intel_engine_pm_put(engine);
>  }
> 
> @@ -189,8 +264,10 @@ void intel_engine_unpark_heartbeat(struct
> intel_engine_cs *engine)
> 
>  void intel_engine_park_heartbeat(struct intel_engine_cs *engine)  {
> -	if (cancel_delayed_work(&engine->heartbeat.work))
> -		i915_request_put(fetch_and_zero(&engine-
> >heartbeat.systole));
> +	/* completion may rearm work */
> +	while (cancel_delayed_work(&engine->heartbeat.work))
> +		;
> +	untrack_heartbeat(engine);
>  }
> 
>  void intel_engine_init_heartbeat(struct intel_engine_cs *engine) diff --git
> a/drivers/gpu/drm/i915/gt/intel_engine_types.h
> b/drivers/gpu/drm/i915/gt/intel_engine_types.h
> index 7159f9575e65..4956594c8b93 100644
> --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
> +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
> @@ -347,6 +347,7 @@ struct intel_engine_cs {
>  	struct {
>  		struct delayed_work work;
>  		struct i915_request *systole;
> +		struct dma_fence_cb cb;
>  		unsigned long blocked;
>  	} heartbeat;
> 
> diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
> b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
> index b2c369317bf1..812c4a168b01 100644
> --- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
> +++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
> @@ -202,47 +202,44 @@ static int cmp_u32(const void *_a, const void *_b)
> 
>  static int __live_heartbeat_fast(struct intel_engine_cs *engine)  {
> -	const unsigned int error_threshold = max(20000u,
> jiffies_to_usecs(6));
> -	struct intel_context *ce;
> -	struct i915_request *rq;
> -	ktime_t t0, t1;
> +	const unsigned int error_threshold =
> +		max(3 * HEARTBEAT_COMPLETION * 1000,
> jiffies_to_usecs(6));
> +	struct intel_context *ce = engine->kernel_context;
>  	u32 times[5];
>  	int err;
>  	int i;
> 
> -	ce = intel_context_create(engine);
> -	if (IS_ERR(ce))
> -		return PTR_ERR(ce);
> -
>  	intel_engine_pm_get(engine);
> 
>  	err = intel_engine_set_heartbeat(engine, 1);
>  	if (err)
>  		goto err_pm;
> 
> +	flush_delayed_work(&engine->heartbeat.work);
> +	while (engine->heartbeat.systole)
> +		intel_engine_park_heartbeat(engine);
> +
>  	for (i = 0; i < ARRAY_SIZE(times); i++) {
> -		do {
> -			/* Manufacture a tick */
> -			intel_engine_park_heartbeat(engine);
> -			GEM_BUG_ON(engine->heartbeat.systole);
> -			engine->serial++; /*  pretend we are not idle! */
> -			intel_engine_unpark_heartbeat(engine);
> +		struct i915_sched_attr attr = { .priority =
> I915_PRIORITY_MIN };
> +		struct i915_request *rq;
> +		ktime_t t0, t1;
> 
> -			flush_delayed_work(&engine->heartbeat.work);
> -			if (!delayed_work_pending(&engine-
> >heartbeat.work)) {
> -				pr_err("%s: heartbeat %d did not start\n",
> -				       engine->name, i);
> -				err = -EINVAL;
> -				goto err_pm;
> -			}
> +		GEM_BUG_ON(READ_ONCE(engine->heartbeat.systole));
> 
> -			rcu_read_lock();
> -			rq = READ_ONCE(engine->heartbeat.systole);
> -			if (rq)
> -				rq = i915_request_get_rcu(rq);
> -			rcu_read_unlock();
> -		} while (!rq);
> +		/* Manufacture a tick */
> +		mutex_lock(&ce->timeline->mutex);
> +		rq = heartbeat_create(ce, GFP_KERNEL);
> +		if (!IS_ERR(rq)) {
> +			i915_request_get(rq);
> +			heartbeat_commit(rq, &attr);
> +		}
> +		mutex_unlock(&ce->timeline->mutex);
> +		if (IS_ERR(rq)) {
> +			err = PTR_ERR(rq);
> +			goto err_reset;
> +		}
> 
> +		/* Time how long before the heartbeat monitor checks */
>  		t0 = ktime_get();
>  		while (rq == READ_ONCE(engine->heartbeat.systole))
>  			yield(); /* work is on the local cpu! */ @@ -275,10
> +272,10 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
>  		err = -EINVAL;
>  	}
> 
> +err_reset:
>  	reset_heartbeat(engine);
>  err_pm:
>  	intel_engine_pm_put(engine);
> -	intel_context_put(ce);
>  	return err;
>  }
> 
> @@ -308,20 +305,16 @@ static int __live_heartbeat_off(struct
> intel_engine_cs *engine)
> 
>  	intel_engine_pm_get(engine);
> 
> +	/* Kick once, so that we change an active heartbeat */
>  	engine->serial++;
> -	flush_delayed_work(&engine->heartbeat.work);
> -	if (!delayed_work_pending(&engine->heartbeat.work)) {
> -		pr_err("%s: heartbeat not running\n",
> -		       engine->name);
> -		err = -EINVAL;
> -		goto err_pm;
> -	}
> +	intel_engine_unpark_heartbeat(engine);
> 
>  	err = intel_engine_set_heartbeat(engine, 0);
>  	if (err)
>  		goto err_pm;
> 
> -	engine->serial++;
> +	/* The next heartbeat work should cancel the heartbeat */
> +	engine->serial++; /* pretend the engine is still active */
>  	flush_delayed_work(&engine->heartbeat.work);
>  	if (delayed_work_pending(&engine->heartbeat.work)) {
>  		pr_err("%s: heartbeat still running\n", diff --git
> a/drivers/gpu/drm/i915/gt/selftest_execlists.c
> b/drivers/gpu/drm/i915/gt/selftest_execlists.c
> index 7bab147e4421..4f7298855278 100644
> --- a/drivers/gpu/drm/i915/gt/selftest_execlists.c
> +++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c
> @@ -2324,13 +2324,16 @@ static int __cancel_fail(struct
> live_preempt_cancel *arg)
>  	del_timer_sync(&engine->execlists.preempt);
>  	intel_engine_flush_submission(engine);
> 
> +	engine->props.preempt_timeout_ms = 0;
>  	cancel_reset_timeout(engine);
> 
> -	/* after failure, require heartbeats to reset device */
> +	/* after failure, require fast heartbeats to reset device */
>  	intel_engine_set_heartbeat(engine, 1);
>  	err = wait_for_reset(engine, rq, HZ / 2);
>  	intel_engine_set_heartbeat(engine,
>  				   engine->defaults.heartbeat_interval_ms);
> +
> +	engine->props.preempt_timeout_ms =
> +engine->defaults.preempt_timeout_ms;
>  	if (err) {
>  		pr_err("Cancelled inflight0 request did not reset\n");
>  		goto out;
> --
> 2.20.1

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply	[flat|nested] 21+ messages in thread

* [Intel-gfx] [PATCH v3] drm/i915/gt: Ratelimit heartbeat completion probing
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (5 preceding siblings ...)
  2021-02-06  6:45 ` [Intel-gfx] ✓ Fi.CI.IGT: " Patchwork
@ 2021-02-11 12:24 ` Chris Wilson
  2021-02-11 14:59 ` [Intel-gfx] ✗ Fi.CI.BAT: failure for drm/i915/gt: Ratelimit heartbeat completion probing (rev8) Patchwork
                   ` (8 subsequent siblings)
  15 siblings, 0 replies; 21+ messages in thread
From: Chris Wilson @ 2021-02-11 12:24 UTC (permalink / raw)
  To: intel-gfx; +Cc: Chris Wilson

The heartbeat runs through a few phases that we expect to complete
within a certain number of heartbeat intervals. First we must submit the
heartbeat to the queue, and if the queue is occupied it may take a
couple of intervals before the heartbeat preempts the workload and is
submitted to HW. Once running on HW, completion is not instantaneous as
it may have to first reset the current workload before it itself runs
through the empty request and signals completion. As such, we know that
the heartbeat must take at least the preempt reset timeout and before we
have had a chance to reset the engine, we do not want to issue a global
reset ourselves (simply so that we only try to do one reset at a time
and not confuse ourselves by resetting twice and hitting an innocent.)

So by taking into consideration that once running the request must take
a finite amount of time, we can delay the final completion check to
accommodate that and avoid checking too early (before we've had a chance
to handle any engine resets required).

v2: Attach a callback to flush the work immediately upon the heartbeat
completion and insert the delay before the next.

v3: Now with more tracking for selftests and detection of
false/unexpected hang reports.

Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/2853
Suggested-by: CQ Tang <cq.tang@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
 .../gpu/drm/i915/gt/intel_engine_heartbeat.c  | 104 +++++++++++++++---
 drivers/gpu/drm/i915/gt/intel_engine_types.h  |   8 ++
 .../drm/i915/gt/selftest_engine_heartbeat.c   |  92 ++++++++++------
 drivers/gpu/drm/i915/gt/selftest_execlists.c  |   5 +-
 4 files changed, 159 insertions(+), 50 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
index 0b062fad1837..71aa02995417 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
@@ -20,6 +20,18 @@
  * issue a reset -- in the hope that restores progress.
  */
 
+#define HEARTBEAT_COMPLETION 50u /* milliseconds */
+
+static long completion_timeout(const struct intel_engine_cs *engine)
+{
+	long timeout = HEARTBEAT_COMPLETION;
+
+	if (intel_engine_has_preempt_reset(engine))
+		timeout += READ_ONCE(engine->props.preempt_timeout_ms);
+
+	return msecs_to_jiffies(timeout);
+}
+
 static bool next_heartbeat(struct intel_engine_cs *engine)
 {
 	long delay;
@@ -29,6 +41,26 @@ static bool next_heartbeat(struct intel_engine_cs *engine)
 		return false;
 
 	delay = msecs_to_jiffies_timeout(delay);
+
+	/*
+	 * Once we submit a heartbeat to the HW, we know that it will take
+	 * at least a certain amount of time to complete. On a hanging system
+	 * it will first have to wait for the preempt reset timeout, and
+	 * then it will take some time for the reset to resume with the
+	 * heartbeat and for it to complete. So once we have submitted the
+	 * heartbeat to HW, we can wait a while longer before declaring the
+	 * engine stuck and forcing a reset ourselves. If we do a reset
+	 * and the engine is also doing a reset, it is possible that we
+	 * reset the engine twice, harming an innocent.
+	 *
+	 * Before we have sumitted the heartbeat, we do not want to change
+	 * the interval as we want to promote the heartbeat and trigger
+	 * preemption in a deterministic time frame.
+	 */
+	if (engine->heartbeat.systole &&
+	    i915_request_is_active(engine->heartbeat.systole))
+		delay = max(delay, completion_timeout(engine));
+
 	if (delay >= HZ)
 		delay = round_jiffies_up_relative(delay);
 	mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, delay + 1);
@@ -48,12 +80,51 @@ heartbeat_create(struct intel_context *ce, gfp_t gfp)
 	return rq;
 }
 
+static void defibrillator(struct dma_fence *f, struct dma_fence_cb *cb)
+{
+	struct intel_engine_cs *engine =
+		container_of(cb, typeof(*engine), heartbeat.cb);
+
+	if (READ_ONCE(engine->heartbeat.systole))
+		mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, 0);
+}
+
+static void
+untrack_heartbeat(struct intel_engine_cs *engine)
+{
+	struct i915_request *rq;
+
+	rq = fetch_and_zero(&engine->heartbeat.systole);
+	if (!rq)
+		return;
+
+	ENGINE_TRACE(engine, "heartbeat " RQ_FMT "completed\n", RQ_ARG(rq));
+	I915_SELFTEST_ONLY(engine->heartbeat.completed++);
+
+	dma_fence_remove_callback(&rq->fence, &engine->heartbeat.cb);
+	i915_request_put(rq);
+}
+
+static void
+track_heartbeat(struct intel_engine_cs *engine, struct i915_request *rq)
+{
+	ENGINE_TRACE(engine, "heartbeat " RQ_FMT "started\n", RQ_ARG(rq));
+	I915_SELFTEST_ONLY(engine->heartbeat.submitted++);
+
+	dma_fence_add_callback(&rq->fence,
+			       &engine->heartbeat.cb,
+			       defibrillator);
+	engine->heartbeat.systole = i915_request_get(rq);
+	if (!next_heartbeat(engine))
+		untrack_heartbeat(engine);
+}
+
 static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq)
 {
 	engine->wakeref_serial = READ_ONCE(engine->serial) + 1;
 	i915_request_add_active_barriers(rq);
 	if (!engine->heartbeat.systole && intel_engine_has_heartbeat(engine))
-		engine->heartbeat.systole = i915_request_get(rq);
+		track_heartbeat(engine, rq);
 }
 
 static void heartbeat_commit(struct i915_request *rq,
@@ -106,13 +177,8 @@ static void heartbeat(struct work_struct *wrk)
 	intel_engine_flush_scheduler(engine);
 
 	rq = engine->heartbeat.systole;
-	if (rq && i915_request_completed(rq)) {
-		ENGINE_TRACE(engine,
-			     "heartbeat " RQ_FMT "completed\n",
-			     RQ_ARG(rq));
-		i915_request_put(rq);
-		engine->heartbeat.systole = NULL;
-	}
+	if (rq && i915_request_completed(rq))
+		untrack_heartbeat(engine);
 
 	if (!intel_engine_pm_get_if_awake(engine))
 		return;
@@ -120,6 +186,8 @@ static void heartbeat(struct work_struct *wrk)
 	if (intel_gt_is_wedged(engine->gt))
 		goto out;
 
+	I915_SELFTEST_ONLY(engine->heartbeat.ticks++);
+
 	if (engine->heartbeat.systole) {
 		long delay = READ_ONCE(engine->props.heartbeat_interval_ms);
 
@@ -152,8 +220,10 @@ static void heartbeat(struct work_struct *wrk)
 			attr.priority = 0;
 			if (rq->sched.attr.priority >= attr.priority)
 				attr.priority = I915_PRIORITY_HEARTBEAT;
-			if (rq->sched.attr.priority >= attr.priority)
+			if (rq->sched.attr.priority >= attr.priority) {
+				I915_SELFTEST_ONLY(engine->heartbeat.barrier++);
 				attr.priority = I915_PRIORITY_BARRIER;
+			}
 
 			ENGINE_TRACE(engine,
 				     "bumping heartbeat " RQ_FMT " prio:%d\n",
@@ -166,6 +236,7 @@ static void heartbeat(struct work_struct *wrk)
 			ENGINE_TRACE(engine,
 				     "heartbeat " RQ_FMT " stuck\n",
 				     RQ_ARG(rq));
+			I915_SELFTEST_ONLY(engine->heartbeat.reset++);
 
 			if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 				show_heartbeat(rq, engine);
@@ -180,6 +251,11 @@ static void heartbeat(struct work_struct *wrk)
 		goto out;
 	}
 
+	/* Just completed one heartbeat, wait a tick before the next */
+	if (rq)
+		goto out;
+
+	/* The engine is parking. We can rest until the next user */
 	serial = READ_ONCE(engine->serial);
 	if (engine->wakeref_serial == serial)
 		goto out;
@@ -198,14 +274,14 @@ static void heartbeat(struct work_struct *wrk)
 	if (IS_ERR(rq))
 		goto unlock;
 
-	ENGINE_TRACE(engine, "heartbeat " RQ_FMT "started\n", RQ_ARG(rq));
 	heartbeat_commit(rq, &attr);
 
 unlock:
 	mutex_unlock(&ce->timeline->mutex);
 out:
+	intel_engine_flush_scheduler(engine);
 	if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+		untrack_heartbeat(engine);
 	intel_engine_pm_put(engine);
 }
 
@@ -219,8 +295,10 @@ void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine)
 
 void intel_engine_park_heartbeat(struct intel_engine_cs *engine)
 {
-	if (cancel_delayed_work(&engine->heartbeat.work))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+	/* completion may rearm work */
+	while (cancel_delayed_work(&engine->heartbeat.work))
+		;
+	untrack_heartbeat(engine);
 }
 
 void intel_engine_init_heartbeat(struct intel_engine_cs *engine)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 7efa6290cc3e..5ad8d26e2fa0 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -322,7 +322,15 @@ struct intel_engine_cs {
 	struct {
 		struct delayed_work work;
 		struct i915_request *systole;
+		struct dma_fence_cb cb;
 		unsigned long blocked;
+		I915_SELFTEST_DECLARE(struct {
+			unsigned long submitted;
+			unsigned long completed;
+			unsigned long barrier;
+			unsigned long reset;
+			unsigned long ticks;
+		};)
 	} heartbeat;
 
 	unsigned long serial;
diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
index b2c369317bf1..3784b1852eac 100644
--- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
@@ -202,47 +202,57 @@ static int cmp_u32(const void *_a, const void *_b)
 
 static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 {
-	const unsigned int error_threshold = max(20000u, jiffies_to_usecs(6));
-	struct intel_context *ce;
-	struct i915_request *rq;
-	ktime_t t0, t1;
+	const unsigned int error_threshold =
+		max_t(unsigned int,
+		      3 * completion_timeout(engine) * 1000,
+		      jiffies_to_usecs(6));
+	struct intel_context *ce = engine->kernel_context;
+	unsigned long resets;
 	u32 times[5];
 	int err;
 	int i;
 
-	ce = intel_context_create(engine);
-	if (IS_ERR(ce))
-		return PTR_ERR(ce);
-
 	intel_engine_pm_get(engine);
 
 	err = intel_engine_set_heartbeat(engine, 1);
 	if (err)
 		goto err_pm;
 
+	flush_delayed_work(&engine->heartbeat.work);
+	while (engine->heartbeat.systole)
+		intel_engine_park_heartbeat(engine);
+
+	resets = engine->heartbeat.reset;
 	for (i = 0; i < ARRAY_SIZE(times); i++) {
-		do {
-			/* Manufacture a tick */
-			intel_engine_park_heartbeat(engine);
-			GEM_BUG_ON(engine->heartbeat.systole);
-			engine->serial++; /*  pretend we are not idle! */
-			intel_engine_unpark_heartbeat(engine);
+		struct i915_sched_attr attr = { .priority = I915_PRIORITY_MIN };
+		unsigned long submits = engine->heartbeat.submitted;
+		unsigned long completed = engine->heartbeat.completed;
+		struct i915_request *rq;
+		ktime_t t0, t1;
 
-			flush_delayed_work(&engine->heartbeat.work);
-			if (!delayed_work_pending(&engine->heartbeat.work)) {
-				pr_err("%s: heartbeat %d did not start\n",
-				       engine->name, i);
-				err = -EINVAL;
-				goto err_pm;
-			}
+		GEM_BUG_ON(READ_ONCE(engine->heartbeat.systole));
 
-			rcu_read_lock();
-			rq = READ_ONCE(engine->heartbeat.systole);
-			if (rq)
-				rq = i915_request_get_rcu(rq);
-			rcu_read_unlock();
-		} while (!rq);
+		/* Manufacture a tick */
+		mutex_lock(&ce->timeline->mutex);
+		rq = heartbeat_create(ce, GFP_KERNEL);
+		if (!IS_ERR(rq)) {
+			i915_request_get(rq);
+			heartbeat_commit(rq, &attr);
+		}
+		mutex_unlock(&ce->timeline->mutex);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto err_reset;
+		}
 
+		if (engine->heartbeat.submitted != submits + 1) {
+			GEM_TRACE_ERR("Heartbeat was not tracked!\n");
+			GEM_TRACE_DUMP();
+			err = -EIO;
+			goto err_reset;
+		}
+
+		/* Time how long before the heartbeat monitor checks */
 		t0 = ktime_get();
 		while (rq == READ_ONCE(engine->heartbeat.systole))
 			yield(); /* work is on the local cpu! */
@@ -250,6 +260,20 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 
 		i915_request_put(rq);
 		times[i] = ktime_us_delta(t1, t0);
+
+		if (engine->heartbeat.reset != resets) {
+			GEM_TRACE_ERR("False reset from a fast heartbeat!\n");
+			GEM_TRACE_DUMP();
+			err = -EIO;
+			goto err_reset;
+		}
+
+		if (engine->heartbeat.completed != completed + 1) {
+			GEM_TRACE_ERR("Completed heartbeat was not recorded!\n");
+			GEM_TRACE_DUMP();
+			err = -EIO;
+			goto err_reset;
+		}
 	}
 
 	sort(times, ARRAY_SIZE(times), sizeof(times[0]), cmp_u32, NULL);
@@ -275,10 +299,10 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 		err = -EINVAL;
 	}
 
+err_reset:
 	reset_heartbeat(engine);
 err_pm:
 	intel_engine_pm_put(engine);
-	intel_context_put(ce);
 	return err;
 }
 
@@ -308,20 +332,16 @@ static int __live_heartbeat_off(struct intel_engine_cs *engine)
 
 	intel_engine_pm_get(engine);
 
+	/* Kick once, so that we change an active heartbeat */
 	engine->serial++;
-	flush_delayed_work(&engine->heartbeat.work);
-	if (!delayed_work_pending(&engine->heartbeat.work)) {
-		pr_err("%s: heartbeat not running\n",
-		       engine->name);
-		err = -EINVAL;
-		goto err_pm;
-	}
+	intel_engine_unpark_heartbeat(engine);
 
 	err = intel_engine_set_heartbeat(engine, 0);
 	if (err)
 		goto err_pm;
 
-	engine->serial++;
+	/* The next heartbeat work should cancel the heartbeat */
+	engine->serial++; /* pretend the engine is still active */
 	flush_delayed_work(&engine->heartbeat.work);
 	if (delayed_work_pending(&engine->heartbeat.work)) {
 		pr_err("%s: heartbeat still running\n",
diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c
index f625c29023ea..04ded3a2d491 100644
--- a/drivers/gpu/drm/i915/gt/selftest_execlists.c
+++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c
@@ -2325,13 +2325,16 @@ static int __cancel_fail(struct live_preempt_cancel *arg)
 	del_timer_sync(&engine->execlists.preempt);
 	intel_engine_flush_scheduler(engine);
 
+	engine->props.preempt_timeout_ms = 0;
 	cancel_reset_timeout(engine);
 
-	/* after failure, require heartbeats to reset device */
+	/* after failure, require fast heartbeats to reset device */
 	intel_engine_set_heartbeat(engine, 1);
 	err = wait_for_reset(engine, rq, HZ / 2);
 	intel_engine_set_heartbeat(engine,
 				   engine->defaults.heartbeat_interval_ms);
+
+	engine->props.preempt_timeout_ms = engine->defaults.preempt_timeout_ms;
 	if (err) {
 		pr_err("Cancelled inflight0 request did not reset\n");
 		goto out;
-- 
2.20.1

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply related	[flat|nested] 21+ messages in thread

* [Intel-gfx] ✗ Fi.CI.BAT: failure for drm/i915/gt: Ratelimit heartbeat completion probing (rev8)
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (6 preceding siblings ...)
  2021-02-11 12:24 ` [Intel-gfx] [PATCH v3] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
@ 2021-02-11 14:59 ` Patchwork
  2021-02-11 21:06 ` [Intel-gfx] [PATCH v4] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (7 subsequent siblings)
  15 siblings, 0 replies; 21+ messages in thread
From: Patchwork @ 2021-02-11 14:59 UTC (permalink / raw)
  To: Chris Wilson; +Cc: intel-gfx


[-- Attachment #1.1: Type: text/plain, Size: 6141 bytes --]

== Series Details ==

Series: drm/i915/gt: Ratelimit heartbeat completion probing (rev8)
URL   : https://patchwork.freedesktop.org/series/86665/
State : failure

== Summary ==

CI Bug Log - changes from CI_DRM_9761 -> Patchwork_19663
====================================================

Summary
-------

  **FAILURE**

  Serious unknown changes coming with Patchwork_19663 absolutely need to be
  verified manually.
  
  If you think the reported changes have nothing to do with the changes
  introduced in Patchwork_19663, please notify your bug team to allow them
  to document this new failure mode, which will reduce false positives in CI.

  External URL: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19663/index.html

Possible new issues
-------------------

  Here are the unknown changes that may have been introduced in Patchwork_19663:

### IGT changes ###

#### Possible regressions ####

  * igt@i915_selftest@live@gt_heartbeat:
    - fi-bdw-5557u:       [PASS][1] -> [INCOMPLETE][2]
   [1]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9761/fi-bdw-5557u/igt@i915_selftest@live@gt_heartbeat.html
   [2]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19663/fi-bdw-5557u/igt@i915_selftest@live@gt_heartbeat.html

  
#### Suppressed ####

  The following results come from untrusted machines, tests, or statuses.
  They do not affect the overall result.

  * igt@i915_selftest@live@gt_heartbeat:
    - {fi-ehl-1}:         [PASS][3] -> [INCOMPLETE][4]
   [3]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9761/fi-ehl-1/igt@i915_selftest@live@gt_heartbeat.html
   [4]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19663/fi-ehl-1/igt@i915_selftest@live@gt_heartbeat.html

  
Known issues
------------

  Here are the changes found in Patchwork_19663 that come from known issues:

### IGT changes ###

#### Issues hit ####

  * igt@amdgpu/amd_basic@memory-alloc:
    - fi-tgl-y:           NOTRUN -> [SKIP][5] ([fdo#109315] / [i915#2575]) +2 similar issues
   [5]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19663/fi-tgl-y/igt@amdgpu/amd_basic@memory-alloc.html

  * igt@gem_exec_suspend@basic-s3:
    - fi-tgl-y:           [PASS][6] -> [DMESG-WARN][7] ([i915#2411] / [i915#402])
   [6]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9761/fi-tgl-y/igt@gem_exec_suspend@basic-s3.html
   [7]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19663/fi-tgl-y/igt@gem_exec_suspend@basic-s3.html

  * igt@i915_selftest@live@gt_heartbeat:
    - fi-hsw-4770:        [PASS][8] -> [INCOMPLETE][9] ([i915#2782])
   [8]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9761/fi-hsw-4770/igt@i915_selftest@live@gt_heartbeat.html
   [9]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19663/fi-hsw-4770/igt@i915_selftest@live@gt_heartbeat.html
    - fi-ivb-3770:        [PASS][10] -> [INCOMPLETE][11] ([i915#2782])
   [10]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9761/fi-ivb-3770/igt@i915_selftest@live@gt_heartbeat.html
   [11]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19663/fi-ivb-3770/igt@i915_selftest@live@gt_heartbeat.html
    - fi-tgl-u2:          [PASS][12] -> [INCOMPLETE][13] ([i915#2601])
   [12]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9761/fi-tgl-u2/igt@i915_selftest@live@gt_heartbeat.html
   [13]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19663/fi-tgl-u2/igt@i915_selftest@live@gt_heartbeat.html

  * igt@prime_self_import@basic-with_one_bo_two_files:
    - fi-tgl-y:           [PASS][14] -> [DMESG-WARN][15] ([i915#402]) +1 similar issue
   [14]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9761/fi-tgl-y/igt@prime_self_import@basic-with_one_bo_two_files.html
   [15]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19663/fi-tgl-y/igt@prime_self_import@basic-with_one_bo_two_files.html

  * igt@runner@aborted:
    - fi-bdw-5557u:       NOTRUN -> [FAIL][16] ([i915#2369])
   [16]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19663/fi-bdw-5557u/igt@runner@aborted.html
    - fi-hsw-4770:        NOTRUN -> [FAIL][17] ([i915#1436] / [i915#2295] / [i915#2505])
   [17]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19663/fi-hsw-4770/igt@runner@aborted.html

  
#### Possible fixes ####

  * igt@prime_self_import@basic-with_two_bos:
    - fi-tgl-y:           [DMESG-WARN][18] ([i915#402]) -> [PASS][19] +1 similar issue
   [18]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9761/fi-tgl-y/igt@prime_self_import@basic-with_two_bos.html
   [19]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19663/fi-tgl-y/igt@prime_self_import@basic-with_two_bos.html

  
  {name}: This element is suppressed. This means it is ignored when computing
          the status of the difference (SUCCESS, WARNING, or FAILURE).

  [fdo#109315]: https://bugs.freedesktop.org/show_bug.cgi?id=109315
  [i915#1436]: https://gitlab.freedesktop.org/drm/intel/issues/1436
  [i915#2295]: https://gitlab.freedesktop.org/drm/intel/issues/2295
  [i915#2369]: https://gitlab.freedesktop.org/drm/intel/issues/2369
  [i915#2411]: https://gitlab.freedesktop.org/drm/intel/issues/2411
  [i915#2505]: https://gitlab.freedesktop.org/drm/intel/issues/2505
  [i915#2575]: https://gitlab.freedesktop.org/drm/intel/issues/2575
  [i915#2601]: https://gitlab.freedesktop.org/drm/intel/issues/2601
  [i915#2782]: https://gitlab.freedesktop.org/drm/intel/issues/2782
  [i915#402]: https://gitlab.freedesktop.org/drm/intel/issues/402


Participating hosts (44 -> 39)
------------------------------

  Missing    (5): fi-ilk-m540 fi-hsw-4200u fi-skl-guc fi-bsw-cyan fi-bdw-samus 


Build changes
-------------

  * Linux: CI_DRM_9761 -> Patchwork_19663

  CI-20190529: 20190529
  CI_DRM_9761: fc52fc2a7332bd301f802ca3a0444a8fb9fe4f7f @ git://anongit.freedesktop.org/gfx-ci/linux
  IGT_6001: d0d6f5e14ef181c93e4b503b05d9c18fa480e09d @ git://anongit.freedesktop.org/xorg/app/intel-gpu-tools
  Patchwork_19663: 19ac78f67144787682baf72dedeca526176ad6e4 @ git://anongit.freedesktop.org/gfx-ci/linux


== Linux commits ==

19ac78f67144 drm/i915/gt: Ratelimit heartbeat completion probing

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19663/index.html

[-- Attachment #1.2: Type: text/html, Size: 7201 bytes --]

[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply	[flat|nested] 21+ messages in thread

* [Intel-gfx] [PATCH v4] drm/i915/gt: Ratelimit heartbeat completion probing
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (7 preceding siblings ...)
  2021-02-11 14:59 ` [Intel-gfx] ✗ Fi.CI.BAT: failure for drm/i915/gt: Ratelimit heartbeat completion probing (rev8) Patchwork
@ 2021-02-11 21:06 ` Chris Wilson
  2021-02-11 22:02 ` [Intel-gfx] ✗ Fi.CI.BAT: failure for drm/i915/gt: Ratelimit heartbeat completion probing (rev9) Patchwork
                   ` (6 subsequent siblings)
  15 siblings, 0 replies; 21+ messages in thread
From: Chris Wilson @ 2021-02-11 21:06 UTC (permalink / raw)
  To: intel-gfx; +Cc: Chris Wilson

The heartbeat runs through a few phases that we expect to complete
within a certain number of heartbeat intervals. First we must submit the
heartbeat to the queue, and if the queue is occupied it may take a
couple of intervals before the heartbeat preempts the workload and is
submitted to HW. Once running on HW, completion is not instantaneous as
it may have to first reset the current workload before it itself runs
through the empty request and signals completion. As such, we know that
the heartbeat must take at least the preempt reset timeout and before we
have had a chance to reset the engine, we do not want to issue a global
reset ourselves (simply so that we only try to do one reset at a time
and not confuse ourselves by resetting twice and hitting an innocent.)

So by taking into consideration that once running the request must take
a finite amount of time, we can delay the final completion check to
accommodate that and avoid checking too early (before we've had a chance
to handle any engine resets required).

v2: Attach a callback to flush the work immediately upon the heartbeat
completion and insert the delay before the next.

v3: Now with more tracking for selftests and detection of
false/unexpected hang reports.

Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/2853
Suggested-by: CQ Tang <cq.tang@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
 .../gpu/drm/i915/gt/intel_engine_heartbeat.c  | 107 +++++++++++++++---
 drivers/gpu/drm/i915/gt/intel_engine_types.h  |   8 ++
 .../drm/i915/gt/selftest_engine_heartbeat.c   |  93 +++++++++------
 drivers/gpu/drm/i915/gt/selftest_execlists.c  |   5 +-
 4 files changed, 163 insertions(+), 50 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
index 0b062fad1837..351f58def216 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
@@ -20,6 +20,18 @@
  * issue a reset -- in the hope that restores progress.
  */
 
+#define HEARTBEAT_COMPLETION 50u /* milliseconds */
+
+static long completion_timeout(const struct intel_engine_cs *engine)
+{
+	long timeout = HEARTBEAT_COMPLETION;
+
+	if (intel_engine_has_preempt_reset(engine))
+		timeout += READ_ONCE(engine->props.preempt_timeout_ms);
+
+	return msecs_to_jiffies(timeout);
+}
+
 static bool next_heartbeat(struct intel_engine_cs *engine)
 {
 	long delay;
@@ -29,6 +41,26 @@ static bool next_heartbeat(struct intel_engine_cs *engine)
 		return false;
 
 	delay = msecs_to_jiffies_timeout(delay);
+
+	/*
+	 * Once we submit a heartbeat to the HW, we know that it will take
+	 * at least a certain amount of time to complete. On a hanging system
+	 * it will first have to wait for the preempt reset timeout, and
+	 * then it will take some time for the reset to resume with the
+	 * heartbeat and for it to complete. So once we have submitted the
+	 * heartbeat to HW, we can wait a while longer before declaring the
+	 * engine stuck and forcing a reset ourselves. If we do a reset
+	 * and the engine is also doing a reset, it is possible that we
+	 * reset the engine twice, harming an innocent.
+	 *
+	 * Before we have sumitted the heartbeat, we do not want to change
+	 * the interval as we want to promote the heartbeat and trigger
+	 * preemption in a deterministic time frame.
+	 */
+	if (engine->heartbeat.systole &&
+	    i915_request_is_active(engine->heartbeat.systole))
+		delay = max(delay, completion_timeout(engine));
+
 	if (delay >= HZ)
 		delay = round_jiffies_up_relative(delay);
 	mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, delay + 1);
@@ -48,12 +80,51 @@ heartbeat_create(struct intel_context *ce, gfp_t gfp)
 	return rq;
 }
 
+static void defibrillator(struct dma_fence *f, struct dma_fence_cb *cb)
+{
+	struct intel_engine_cs *engine =
+		container_of(cb, typeof(*engine), heartbeat.cb);
+
+	if (READ_ONCE(engine->heartbeat.systole))
+		mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, 0);
+}
+
+static void
+untrack_heartbeat(struct intel_engine_cs *engine)
+{
+	struct i915_request *rq;
+
+	rq = fetch_and_zero(&engine->heartbeat.systole);
+	if (!rq)
+		return;
+
+	ENGINE_TRACE(engine, "heartbeat " RQ_FMT "completed\n", RQ_ARG(rq));
+	I915_SELFTEST_ONLY(engine->heartbeat.completed++);
+
+	dma_fence_remove_callback(&rq->fence, &engine->heartbeat.cb);
+	i915_request_put(rq);
+}
+
+static void
+track_heartbeat(struct intel_engine_cs *engine, struct i915_request *rq)
+{
+	ENGINE_TRACE(engine, "heartbeat " RQ_FMT "started\n", RQ_ARG(rq));
+	I915_SELFTEST_ONLY(engine->heartbeat.submitted++);
+
+	dma_fence_add_callback(&rq->fence,
+			       &engine->heartbeat.cb,
+			       defibrillator);
+	engine->heartbeat.systole = i915_request_get(rq);
+	if (!next_heartbeat(engine))
+		untrack_heartbeat(engine);
+}
+
 static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq)
 {
 	engine->wakeref_serial = READ_ONCE(engine->serial) + 1;
 	i915_request_add_active_barriers(rq);
 	if (!engine->heartbeat.systole && intel_engine_has_heartbeat(engine))
-		engine->heartbeat.systole = i915_request_get(rq);
+		track_heartbeat(engine, rq);
 }
 
 static void heartbeat_commit(struct i915_request *rq,
@@ -106,13 +177,8 @@ static void heartbeat(struct work_struct *wrk)
 	intel_engine_flush_scheduler(engine);
 
 	rq = engine->heartbeat.systole;
-	if (rq && i915_request_completed(rq)) {
-		ENGINE_TRACE(engine,
-			     "heartbeat " RQ_FMT "completed\n",
-			     RQ_ARG(rq));
-		i915_request_put(rq);
-		engine->heartbeat.systole = NULL;
-	}
+	if (rq && i915_request_completed(rq))
+		untrack_heartbeat(engine);
 
 	if (!intel_engine_pm_get_if_awake(engine))
 		return;
@@ -120,9 +186,14 @@ static void heartbeat(struct work_struct *wrk)
 	if (intel_gt_is_wedged(engine->gt))
 		goto out;
 
+	I915_SELFTEST_ONLY(engine->heartbeat.ticks++);
+
 	if (engine->heartbeat.systole) {
 		long delay = READ_ONCE(engine->props.heartbeat_interval_ms);
 
+		if (i915_request_is_active(engine->heartbeat.systole))
+			delay += completion_timeout(engine);
+
 		/* Safeguard against too-fast worker invocations */
 		if (!time_after(jiffies,
 				rq->emitted_jiffies + msecs_to_jiffies(delay)))
@@ -152,8 +223,10 @@ static void heartbeat(struct work_struct *wrk)
 			attr.priority = 0;
 			if (rq->sched.attr.priority >= attr.priority)
 				attr.priority = I915_PRIORITY_HEARTBEAT;
-			if (rq->sched.attr.priority >= attr.priority)
+			if (rq->sched.attr.priority >= attr.priority) {
+				I915_SELFTEST_ONLY(engine->heartbeat.barrier++);
 				attr.priority = I915_PRIORITY_BARRIER;
+			}
 
 			ENGINE_TRACE(engine,
 				     "bumping heartbeat " RQ_FMT " prio:%d\n",
@@ -166,6 +239,7 @@ static void heartbeat(struct work_struct *wrk)
 			ENGINE_TRACE(engine,
 				     "heartbeat " RQ_FMT " stuck\n",
 				     RQ_ARG(rq));
+			I915_SELFTEST_ONLY(engine->heartbeat.reset++);
 
 			if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 				show_heartbeat(rq, engine);
@@ -180,6 +254,11 @@ static void heartbeat(struct work_struct *wrk)
 		goto out;
 	}
 
+	/* Just completed one heartbeat, wait a tick before the next */
+	if (rq)
+		goto out;
+
+	/* The engine is parking. We can rest until the next user */
 	serial = READ_ONCE(engine->serial);
 	if (engine->wakeref_serial == serial)
 		goto out;
@@ -198,14 +277,14 @@ static void heartbeat(struct work_struct *wrk)
 	if (IS_ERR(rq))
 		goto unlock;
 
-	ENGINE_TRACE(engine, "heartbeat " RQ_FMT "started\n", RQ_ARG(rq));
 	heartbeat_commit(rq, &attr);
 
 unlock:
 	mutex_unlock(&ce->timeline->mutex);
 out:
+	intel_engine_flush_scheduler(engine);
 	if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+		untrack_heartbeat(engine);
 	intel_engine_pm_put(engine);
 }
 
@@ -219,8 +298,10 @@ void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine)
 
 void intel_engine_park_heartbeat(struct intel_engine_cs *engine)
 {
-	if (cancel_delayed_work(&engine->heartbeat.work))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+	/* completion may rearm work */
+	while (cancel_delayed_work(&engine->heartbeat.work))
+		;
+	untrack_heartbeat(engine);
 }
 
 void intel_engine_init_heartbeat(struct intel_engine_cs *engine)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 7efa6290cc3e..5ad8d26e2fa0 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -322,7 +322,15 @@ struct intel_engine_cs {
 	struct {
 		struct delayed_work work;
 		struct i915_request *systole;
+		struct dma_fence_cb cb;
 		unsigned long blocked;
+		I915_SELFTEST_DECLARE(struct {
+			unsigned long submitted;
+			unsigned long completed;
+			unsigned long barrier;
+			unsigned long reset;
+			unsigned long ticks;
+		};)
 	} heartbeat;
 
 	unsigned long serial;
diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
index b2c369317bf1..b74459561fda 100644
--- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
@@ -202,47 +202,58 @@ static int cmp_u32(const void *_a, const void *_b)
 
 static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 {
-	const unsigned int error_threshold = max(20000u, jiffies_to_usecs(6));
-	struct intel_context *ce;
-	struct i915_request *rq;
-	ktime_t t0, t1;
+	const unsigned int error_threshold =
+		max_t(unsigned int,
+		      3 * completion_timeout(engine) * 1000,
+		      jiffies_to_usecs(6));
+	struct intel_context *ce = engine->kernel_context;
+	unsigned long resets;
 	u32 times[5];
 	int err;
 	int i;
 
-	ce = intel_context_create(engine);
-	if (IS_ERR(ce))
-		return PTR_ERR(ce);
-
 	intel_engine_pm_get(engine);
 
 	err = intel_engine_set_heartbeat(engine, 1);
 	if (err)
 		goto err_pm;
 
+	flush_delayed_work(&engine->heartbeat.work);
+	while (engine->heartbeat.systole)
+		intel_engine_park_heartbeat(engine);
+
+	resets = engine->heartbeat.reset;
 	for (i = 0; i < ARRAY_SIZE(times); i++) {
-		do {
-			/* Manufacture a tick */
-			intel_engine_park_heartbeat(engine);
-			GEM_BUG_ON(engine->heartbeat.systole);
-			engine->serial++; /*  pretend we are not idle! */
-			intel_engine_unpark_heartbeat(engine);
+		struct i915_sched_attr attr = { .priority = I915_PRIORITY_MIN };
+		unsigned long submits = engine->heartbeat.submitted;
+		unsigned long completed = engine->heartbeat.completed;
+		struct i915_request *rq;
+		ktime_t t0, t1;
 
-			flush_delayed_work(&engine->heartbeat.work);
-			if (!delayed_work_pending(&engine->heartbeat.work)) {
-				pr_err("%s: heartbeat %d did not start\n",
-				       engine->name, i);
-				err = -EINVAL;
-				goto err_pm;
-			}
+		GEM_BUG_ON(READ_ONCE(engine->heartbeat.systole));
 
-			rcu_read_lock();
-			rq = READ_ONCE(engine->heartbeat.systole);
-			if (rq)
-				rq = i915_request_get_rcu(rq);
-			rcu_read_unlock();
-		} while (!rq);
+		/* Manufacture a tick */
+		mutex_lock(&ce->timeline->mutex);
+		rq = heartbeat_create(ce, GFP_KERNEL);
+		if (!IS_ERR(rq)) {
+			i915_request_get(rq);
+			heartbeat_commit(rq, &attr);
+		}
+		mutex_unlock(&ce->timeline->mutex);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto err_reset;
+		}
 
+		if (engine->heartbeat.submitted != submits + 1) {
+			GEM_TRACE_ERR("Heartbeat was not tracked!\n");
+			GEM_TRACE_DUMP();
+			i915_request_put(rq);
+			err = -EIO;
+			goto err_reset;
+		}
+
+		/* Time how long before the heartbeat monitor checks */
 		t0 = ktime_get();
 		while (rq == READ_ONCE(engine->heartbeat.systole))
 			yield(); /* work is on the local cpu! */
@@ -250,6 +261,20 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 
 		i915_request_put(rq);
 		times[i] = ktime_us_delta(t1, t0);
+
+		if (engine->heartbeat.reset != resets) {
+			GEM_TRACE_ERR("False reset from a fast heartbeat!\n");
+			GEM_TRACE_DUMP();
+			err = -EIO;
+			goto err_reset;
+		}
+
+		if (engine->heartbeat.completed != completed + 1) {
+			GEM_TRACE_ERR("Completed heartbeat was not recorded!\n");
+			GEM_TRACE_DUMP();
+			err = -EIO;
+			goto err_reset;
+		}
 	}
 
 	sort(times, ARRAY_SIZE(times), sizeof(times[0]), cmp_u32, NULL);
@@ -275,10 +300,10 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 		err = -EINVAL;
 	}
 
+err_reset:
 	reset_heartbeat(engine);
 err_pm:
 	intel_engine_pm_put(engine);
-	intel_context_put(ce);
 	return err;
 }
 
@@ -308,20 +333,16 @@ static int __live_heartbeat_off(struct intel_engine_cs *engine)
 
 	intel_engine_pm_get(engine);
 
+	/* Kick once, so that we change an active heartbeat */
 	engine->serial++;
-	flush_delayed_work(&engine->heartbeat.work);
-	if (!delayed_work_pending(&engine->heartbeat.work)) {
-		pr_err("%s: heartbeat not running\n",
-		       engine->name);
-		err = -EINVAL;
-		goto err_pm;
-	}
+	intel_engine_unpark_heartbeat(engine);
 
 	err = intel_engine_set_heartbeat(engine, 0);
 	if (err)
 		goto err_pm;
 
-	engine->serial++;
+	/* The next heartbeat work should cancel the heartbeat */
+	engine->serial++; /* pretend the engine is still active */
 	flush_delayed_work(&engine->heartbeat.work);
 	if (delayed_work_pending(&engine->heartbeat.work)) {
 		pr_err("%s: heartbeat still running\n",
diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c
index 5c8a6ccdf024..660240550097 100644
--- a/drivers/gpu/drm/i915/gt/selftest_execlists.c
+++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c
@@ -2325,13 +2325,16 @@ static int __cancel_fail(struct live_preempt_cancel *arg)
 	del_timer_sync(&engine->execlists.preempt);
 	intel_engine_flush_scheduler(engine);
 
+	engine->props.preempt_timeout_ms = 0;
 	cancel_reset_timeout(engine);
 
-	/* after failure, require heartbeats to reset device */
+	/* after failure, require fast heartbeats to reset device */
 	intel_engine_set_heartbeat(engine, 1);
 	err = wait_for_reset(engine, rq, HZ / 2);
 	intel_engine_set_heartbeat(engine,
 				   engine->defaults.heartbeat_interval_ms);
+
+	engine->props.preempt_timeout_ms = engine->defaults.preempt_timeout_ms;
 	if (err) {
 		pr_err("Cancelled inflight0 request did not reset\n");
 		goto out;
-- 
2.20.1

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply related	[flat|nested] 21+ messages in thread

* [Intel-gfx] ✗ Fi.CI.BAT: failure for drm/i915/gt: Ratelimit heartbeat completion probing (rev9)
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (8 preceding siblings ...)
  2021-02-11 21:06 ` [Intel-gfx] [PATCH v4] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
@ 2021-02-11 22:02 ` Patchwork
  2021-02-11 23:14 ` [Intel-gfx] [PATCH v5] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (5 subsequent siblings)
  15 siblings, 0 replies; 21+ messages in thread
From: Patchwork @ 2021-02-11 22:02 UTC (permalink / raw)
  To: Chris Wilson; +Cc: intel-gfx


[-- Attachment #1.1: Type: text/plain, Size: 6582 bytes --]

== Series Details ==

Series: drm/i915/gt: Ratelimit heartbeat completion probing (rev9)
URL   : https://patchwork.freedesktop.org/series/86665/
State : failure

== Summary ==

CI Bug Log - changes from CI_DRM_9764 -> Patchwork_19666
====================================================

Summary
-------

  **FAILURE**

  Serious unknown changes coming with Patchwork_19666 absolutely need to be
  verified manually.
  
  If you think the reported changes have nothing to do with the changes
  introduced in Patchwork_19666, please notify your bug team to allow them
  to document this new failure mode, which will reduce false positives in CI.

  External URL: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/index.html

Possible new issues
-------------------

  Here are the unknown changes that may have been introduced in Patchwork_19666:

### IGT changes ###

#### Possible regressions ####

  * igt@i915_selftest@live@gt_heartbeat:
    - fi-cfl-8109u:       [PASS][1] -> [INCOMPLETE][2]
   [1]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9764/fi-cfl-8109u/igt@i915_selftest@live@gt_heartbeat.html
   [2]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-cfl-8109u/igt@i915_selftest@live@gt_heartbeat.html
    - fi-bxt-dsi:         [PASS][3] -> [INCOMPLETE][4]
   [3]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9764/fi-bxt-dsi/igt@i915_selftest@live@gt_heartbeat.html
   [4]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-bxt-dsi/igt@i915_selftest@live@gt_heartbeat.html
    - fi-cml-s:           [PASS][5] -> [INCOMPLETE][6]
   [5]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9764/fi-cml-s/igt@i915_selftest@live@gt_heartbeat.html
   [6]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-cml-s/igt@i915_selftest@live@gt_heartbeat.html

  
Known issues
------------

  Here are the changes found in Patchwork_19666 that come from known issues:

### IGT changes ###

#### Issues hit ####

  * igt@amdgpu/amd_cs_nop@sync-gfx0:
    - fi-bsw-n3050:       NOTRUN -> [SKIP][7] ([fdo#109271]) +17 similar issues
   [7]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-bsw-n3050/igt@amdgpu/amd_cs_nop@sync-gfx0.html

  * igt@i915_selftest@live@gt_heartbeat:
    - fi-snb-2600:        [PASS][8] -> [INCOMPLETE][9] ([i915#2782])
   [8]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9764/fi-snb-2600/igt@i915_selftest@live@gt_heartbeat.html
   [9]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-snb-2600/igt@i915_selftest@live@gt_heartbeat.html
    - fi-ivb-3770:        [PASS][10] -> [INCOMPLETE][11] ([i915#2782])
   [10]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9764/fi-ivb-3770/igt@i915_selftest@live@gt_heartbeat.html
   [11]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-ivb-3770/igt@i915_selftest@live@gt_heartbeat.html
    - fi-icl-y:           [PASS][12] -> [INCOMPLETE][13] ([i915#2782])
   [12]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9764/fi-icl-y/igt@i915_selftest@live@gt_heartbeat.html
   [13]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-icl-y/igt@i915_selftest@live@gt_heartbeat.html

  * igt@kms_chamelium@dp-crc-fast:
    - fi-kbl-7500u:       [PASS][14] -> [FAIL][15] ([i915#1372])
   [14]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9764/fi-kbl-7500u/igt@kms_chamelium@dp-crc-fast.html
   [15]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-kbl-7500u/igt@kms_chamelium@dp-crc-fast.html

  * igt@prime_self_import@basic-with_two_bos:
    - fi-tgl-y:           [PASS][16] -> [DMESG-WARN][17] ([i915#402]) +1 similar issue
   [16]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9764/fi-tgl-y/igt@prime_self_import@basic-with_two_bos.html
   [17]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-tgl-y/igt@prime_self_import@basic-with_two_bos.html

  * igt@runner@aborted:
    - fi-cfl-8109u:       NOTRUN -> [FAIL][18] ([i915#2295])
   [18]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-cfl-8109u/igt@runner@aborted.html
    - fi-snb-2600:        NOTRUN -> [FAIL][19] ([i915#698])
   [19]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-snb-2600/igt@runner@aborted.html
    - fi-bxt-dsi:         NOTRUN -> [FAIL][20] ([i915#2295])
   [20]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-bxt-dsi/igt@runner@aborted.html
    - fi-icl-y:           NOTRUN -> [FAIL][21] ([i915#2295] / [i915#2724])
   [21]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-icl-y/igt@runner@aborted.html

  
#### Possible fixes ####

  * igt@i915_selftest@live@execlists:
    - fi-bsw-n3050:       [INCOMPLETE][22] ([i915#2940]) -> [PASS][23]
   [22]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9764/fi-bsw-n3050/igt@i915_selftest@live@execlists.html
   [23]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-bsw-n3050/igt@i915_selftest@live@execlists.html

  * igt@prime_self_import@basic-with_one_bo_two_files:
    - fi-tgl-y:           [DMESG-WARN][24] ([i915#402]) -> [PASS][25] +1 similar issue
   [24]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9764/fi-tgl-y/igt@prime_self_import@basic-with_one_bo_two_files.html
   [25]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/fi-tgl-y/igt@prime_self_import@basic-with_one_bo_two_files.html

  
  [fdo#109271]: https://bugs.freedesktop.org/show_bug.cgi?id=109271
  [i915#1372]: https://gitlab.freedesktop.org/drm/intel/issues/1372
  [i915#2295]: https://gitlab.freedesktop.org/drm/intel/issues/2295
  [i915#2724]: https://gitlab.freedesktop.org/drm/intel/issues/2724
  [i915#2782]: https://gitlab.freedesktop.org/drm/intel/issues/2782
  [i915#2940]: https://gitlab.freedesktop.org/drm/intel/issues/2940
  [i915#402]: https://gitlab.freedesktop.org/drm/intel/issues/402
  [i915#698]: https://gitlab.freedesktop.org/drm/intel/issues/698


Participating hosts (44 -> 39)
------------------------------

  Missing    (5): fi-ilk-m540 fi-hsw-4200u fi-bsw-cyan fi-ctg-p8600 fi-bdw-samus 


Build changes
-------------

  * Linux: CI_DRM_9764 -> Patchwork_19666

  CI-20190529: 20190529
  CI_DRM_9764: ecd8de40423fd2824e7f4b29c081912e4b79720e @ git://anongit.freedesktop.org/gfx-ci/linux
  IGT_6001: d0d6f5e14ef181c93e4b503b05d9c18fa480e09d @ git://anongit.freedesktop.org/xorg/app/intel-gpu-tools
  Patchwork_19666: fdd529d2d7f8b819498bb16ccf4bcc59de1a391d @ git://anongit.freedesktop.org/gfx-ci/linux


== Linux commits ==

fdd529d2d7f8 drm/i915/gt: Ratelimit heartbeat completion probing

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19666/index.html

[-- Attachment #1.2: Type: text/html, Size: 7844 bytes --]

[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply	[flat|nested] 21+ messages in thread

* [Intel-gfx] [PATCH v5] drm/i915/gt: Ratelimit heartbeat completion probing
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (9 preceding siblings ...)
  2021-02-11 22:02 ` [Intel-gfx] ✗ Fi.CI.BAT: failure for drm/i915/gt: Ratelimit heartbeat completion probing (rev9) Patchwork
@ 2021-02-11 23:14 ` Chris Wilson
  2021-02-11 23:54   ` Tang, CQ
  2021-02-12  2:58 ` [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev10) Patchwork
                   ` (4 subsequent siblings)
  15 siblings, 1 reply; 21+ messages in thread
From: Chris Wilson @ 2021-02-11 23:14 UTC (permalink / raw)
  To: intel-gfx; +Cc: Chris Wilson

The heartbeat runs through a few phases that we expect to complete
within a certain number of heartbeat intervals. First we must submit the
heartbeat to the queue, and if the queue is occupied it may take a
couple of intervals before the heartbeat preempts the workload and is
submitted to HW. Once running on HW, completion is not instantaneous as
it may have to first reset the current workload before it itself runs
through the empty request and signals completion. As such, we know that
the heartbeat must take at least the preempt reset timeout and before we
have had a chance to reset the engine, we do not want to issue a global
reset ourselves (simply so that we only try to do one reset at a time
and not confuse ourselves by resetting twice and hitting an innocent.)

So by taking into consideration that once running the request must take
a finite amount of time, we can delay the final completion check to
accommodate that and avoid checking too early (before we've had a chance
to handle any engine resets required).

v3: Now with more tracking for selftests and detection of
false/unexpected hang reports.

Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/2853
Suggested-by: CQ Tang <cq.tang@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
 .../gpu/drm/i915/gt/intel_engine_heartbeat.c  | 92 +++++++++++++++---
 drivers/gpu/drm/i915/gt/intel_engine_types.h  |  7 ++
 .../drm/i915/gt/selftest_engine_heartbeat.c   | 93 ++++++++++++-------
 drivers/gpu/drm/i915/gt/selftest_execlists.c  |  5 +-
 4 files changed, 148 insertions(+), 49 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
index 0b062fad1837..e14f9eab5779 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
@@ -20,6 +20,18 @@
  * issue a reset -- in the hope that restores progress.
  */
 
+#define HEARTBEAT_COMPLETION 50u /* milliseconds */
+
+static long completion_timeout(const struct intel_engine_cs *engine)
+{
+	long timeout = HEARTBEAT_COMPLETION;
+
+	if (intel_engine_has_preempt_reset(engine))
+		timeout += READ_ONCE(engine->props.preempt_timeout_ms);
+
+	return msecs_to_jiffies(timeout);
+}
+
 static bool next_heartbeat(struct intel_engine_cs *engine)
 {
 	long delay;
@@ -29,6 +41,26 @@ static bool next_heartbeat(struct intel_engine_cs *engine)
 		return false;
 
 	delay = msecs_to_jiffies_timeout(delay);
+
+	/*
+	 * Once we submit a heartbeat to the HW, we know that it will take
+	 * at least a certain amount of time to complete. On a hanging system
+	 * it will first have to wait for the preempt reset timeout, and
+	 * then it will take some time for the reset to resume with the
+	 * heartbeat and for it to complete. So once we have submitted the
+	 * heartbeat to HW, we can wait a while longer before declaring the
+	 * engine stuck and forcing a reset ourselves. If we do a reset
+	 * and the engine is also doing a reset, it is possible that we
+	 * reset the engine twice, harming an innocent.
+	 *
+	 * Before we have sumitted the heartbeat, we do not want to change
+	 * the interval as we want to promote the heartbeat and trigger
+	 * preemption in a deterministic time frame.
+	 */
+	if (engine->heartbeat.systole &&
+	    i915_request_is_active(engine->heartbeat.systole))
+		delay = max(delay, completion_timeout(engine));
+
 	if (delay >= HZ)
 		delay = round_jiffies_up_relative(delay);
 	mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, delay + 1);
@@ -48,12 +80,39 @@ heartbeat_create(struct intel_context *ce, gfp_t gfp)
 	return rq;
 }
 
+static void
+untrack_heartbeat(struct intel_engine_cs *engine)
+{
+	struct i915_request *rq;
+
+	rq = engine->heartbeat.systole;
+	if (!rq)
+		return;
+
+	ENGINE_TRACE(engine, "heartbeat " RQ_FMT " completed\n", RQ_ARG(rq));
+	I915_SELFTEST_ONLY(engine->heartbeat.completed++);
+
+	WRITE_ONCE(engine->heartbeat.systole, NULL);
+	i915_request_put(rq);
+}
+
+static void
+track_heartbeat(struct intel_engine_cs *engine, struct i915_request *rq)
+{
+	ENGINE_TRACE(engine, "heartbeat " RQ_FMT " started\n", RQ_ARG(rq));
+	I915_SELFTEST_ONLY(engine->heartbeat.submitted++);
+
+	WRITE_ONCE(engine->heartbeat.systole, i915_request_get(rq));
+	if (!next_heartbeat(engine))
+		untrack_heartbeat(engine);
+}
+
 static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq)
 {
 	engine->wakeref_serial = READ_ONCE(engine->serial) + 1;
 	i915_request_add_active_barriers(rq);
 	if (!engine->heartbeat.systole && intel_engine_has_heartbeat(engine))
-		engine->heartbeat.systole = i915_request_get(rq);
+		track_heartbeat(engine, rq);
 }
 
 static void heartbeat_commit(struct i915_request *rq,
@@ -106,13 +165,8 @@ static void heartbeat(struct work_struct *wrk)
 	intel_engine_flush_scheduler(engine);
 
 	rq = engine->heartbeat.systole;
-	if (rq && i915_request_completed(rq)) {
-		ENGINE_TRACE(engine,
-			     "heartbeat " RQ_FMT "completed\n",
-			     RQ_ARG(rq));
-		i915_request_put(rq);
-		engine->heartbeat.systole = NULL;
-	}
+	if (rq && i915_request_completed(rq))
+		untrack_heartbeat(engine);
 
 	if (!intel_engine_pm_get_if_awake(engine))
 		return;
@@ -120,9 +174,15 @@ static void heartbeat(struct work_struct *wrk)
 	if (intel_gt_is_wedged(engine->gt))
 		goto out;
 
+	I915_SELFTEST_ONLY(engine->heartbeat.ticks++);
+
 	if (engine->heartbeat.systole) {
 		long delay = READ_ONCE(engine->props.heartbeat_interval_ms);
 
+		if (rq->sched.attr.priority == I915_PRIORITY_BARRIER &&
+		    i915_request_is_active(rq))
+			delay += completion_timeout(engine);
+
 		/* Safeguard against too-fast worker invocations */
 		if (!time_after(jiffies,
 				rq->emitted_jiffies + msecs_to_jiffies(delay)))
@@ -152,8 +212,10 @@ static void heartbeat(struct work_struct *wrk)
 			attr.priority = 0;
 			if (rq->sched.attr.priority >= attr.priority)
 				attr.priority = I915_PRIORITY_HEARTBEAT;
-			if (rq->sched.attr.priority >= attr.priority)
+			if (rq->sched.attr.priority >= attr.priority) {
+				I915_SELFTEST_ONLY(engine->heartbeat.barrier++);
 				attr.priority = I915_PRIORITY_BARRIER;
+			}
 
 			ENGINE_TRACE(engine,
 				     "bumping heartbeat " RQ_FMT " prio:%d\n",
@@ -166,6 +228,7 @@ static void heartbeat(struct work_struct *wrk)
 			ENGINE_TRACE(engine,
 				     "heartbeat " RQ_FMT " stuck\n",
 				     RQ_ARG(rq));
+			I915_SELFTEST_ONLY(engine->heartbeat.reset++);
 
 			if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 				show_heartbeat(rq, engine);
@@ -180,6 +243,11 @@ static void heartbeat(struct work_struct *wrk)
 		goto out;
 	}
 
+	/* Just completed one heartbeat, wait a tick before the next */
+	if (rq)
+		goto out;
+
+	/* The engine is parking. We can rest until the next user */
 	serial = READ_ONCE(engine->serial);
 	if (engine->wakeref_serial == serial)
 		goto out;
@@ -198,14 +266,14 @@ static void heartbeat(struct work_struct *wrk)
 	if (IS_ERR(rq))
 		goto unlock;
 
-	ENGINE_TRACE(engine, "heartbeat " RQ_FMT "started\n", RQ_ARG(rq));
 	heartbeat_commit(rq, &attr);
 
 unlock:
 	mutex_unlock(&ce->timeline->mutex);
 out:
+	intel_engine_flush_scheduler(engine);
 	if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+		untrack_heartbeat(engine);
 	intel_engine_pm_put(engine);
 }
 
@@ -220,7 +288,7 @@ void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine)
 void intel_engine_park_heartbeat(struct intel_engine_cs *engine)
 {
 	if (cancel_delayed_work(&engine->heartbeat.work))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+		untrack_heartbeat(engine);
 }
 
 void intel_engine_init_heartbeat(struct intel_engine_cs *engine)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 7efa6290cc3e..1e151d550bb5 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -323,6 +323,13 @@ struct intel_engine_cs {
 		struct delayed_work work;
 		struct i915_request *systole;
 		unsigned long blocked;
+		I915_SELFTEST_DECLARE(struct {
+			unsigned long submitted;
+			unsigned long completed;
+			unsigned long barrier;
+			unsigned long reset;
+			unsigned long ticks;
+		};)
 	} heartbeat;
 
 	unsigned long serial;
diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
index b2c369317bf1..b74459561fda 100644
--- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
@@ -202,47 +202,58 @@ static int cmp_u32(const void *_a, const void *_b)
 
 static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 {
-	const unsigned int error_threshold = max(20000u, jiffies_to_usecs(6));
-	struct intel_context *ce;
-	struct i915_request *rq;
-	ktime_t t0, t1;
+	const unsigned int error_threshold =
+		max_t(unsigned int,
+		      3 * completion_timeout(engine) * 1000,
+		      jiffies_to_usecs(6));
+	struct intel_context *ce = engine->kernel_context;
+	unsigned long resets;
 	u32 times[5];
 	int err;
 	int i;
 
-	ce = intel_context_create(engine);
-	if (IS_ERR(ce))
-		return PTR_ERR(ce);
-
 	intel_engine_pm_get(engine);
 
 	err = intel_engine_set_heartbeat(engine, 1);
 	if (err)
 		goto err_pm;
 
+	flush_delayed_work(&engine->heartbeat.work);
+	while (engine->heartbeat.systole)
+		intel_engine_park_heartbeat(engine);
+
+	resets = engine->heartbeat.reset;
 	for (i = 0; i < ARRAY_SIZE(times); i++) {
-		do {
-			/* Manufacture a tick */
-			intel_engine_park_heartbeat(engine);
-			GEM_BUG_ON(engine->heartbeat.systole);
-			engine->serial++; /*  pretend we are not idle! */
-			intel_engine_unpark_heartbeat(engine);
+		struct i915_sched_attr attr = { .priority = I915_PRIORITY_MIN };
+		unsigned long submits = engine->heartbeat.submitted;
+		unsigned long completed = engine->heartbeat.completed;
+		struct i915_request *rq;
+		ktime_t t0, t1;
 
-			flush_delayed_work(&engine->heartbeat.work);
-			if (!delayed_work_pending(&engine->heartbeat.work)) {
-				pr_err("%s: heartbeat %d did not start\n",
-				       engine->name, i);
-				err = -EINVAL;
-				goto err_pm;
-			}
+		GEM_BUG_ON(READ_ONCE(engine->heartbeat.systole));
 
-			rcu_read_lock();
-			rq = READ_ONCE(engine->heartbeat.systole);
-			if (rq)
-				rq = i915_request_get_rcu(rq);
-			rcu_read_unlock();
-		} while (!rq);
+		/* Manufacture a tick */
+		mutex_lock(&ce->timeline->mutex);
+		rq = heartbeat_create(ce, GFP_KERNEL);
+		if (!IS_ERR(rq)) {
+			i915_request_get(rq);
+			heartbeat_commit(rq, &attr);
+		}
+		mutex_unlock(&ce->timeline->mutex);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto err_reset;
+		}
 
+		if (engine->heartbeat.submitted != submits + 1) {
+			GEM_TRACE_ERR("Heartbeat was not tracked!\n");
+			GEM_TRACE_DUMP();
+			i915_request_put(rq);
+			err = -EIO;
+			goto err_reset;
+		}
+
+		/* Time how long before the heartbeat monitor checks */
 		t0 = ktime_get();
 		while (rq == READ_ONCE(engine->heartbeat.systole))
 			yield(); /* work is on the local cpu! */
@@ -250,6 +261,20 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 
 		i915_request_put(rq);
 		times[i] = ktime_us_delta(t1, t0);
+
+		if (engine->heartbeat.reset != resets) {
+			GEM_TRACE_ERR("False reset from a fast heartbeat!\n");
+			GEM_TRACE_DUMP();
+			err = -EIO;
+			goto err_reset;
+		}
+
+		if (engine->heartbeat.completed != completed + 1) {
+			GEM_TRACE_ERR("Completed heartbeat was not recorded!\n");
+			GEM_TRACE_DUMP();
+			err = -EIO;
+			goto err_reset;
+		}
 	}
 
 	sort(times, ARRAY_SIZE(times), sizeof(times[0]), cmp_u32, NULL);
@@ -275,10 +300,10 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 		err = -EINVAL;
 	}
 
+err_reset:
 	reset_heartbeat(engine);
 err_pm:
 	intel_engine_pm_put(engine);
-	intel_context_put(ce);
 	return err;
 }
 
@@ -308,20 +333,16 @@ static int __live_heartbeat_off(struct intel_engine_cs *engine)
 
 	intel_engine_pm_get(engine);
 
+	/* Kick once, so that we change an active heartbeat */
 	engine->serial++;
-	flush_delayed_work(&engine->heartbeat.work);
-	if (!delayed_work_pending(&engine->heartbeat.work)) {
-		pr_err("%s: heartbeat not running\n",
-		       engine->name);
-		err = -EINVAL;
-		goto err_pm;
-	}
+	intel_engine_unpark_heartbeat(engine);
 
 	err = intel_engine_set_heartbeat(engine, 0);
 	if (err)
 		goto err_pm;
 
-	engine->serial++;
+	/* The next heartbeat work should cancel the heartbeat */
+	engine->serial++; /* pretend the engine is still active */
 	flush_delayed_work(&engine->heartbeat.work);
 	if (delayed_work_pending(&engine->heartbeat.work)) {
 		pr_err("%s: heartbeat still running\n",
diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c
index 5c8a6ccdf024..660240550097 100644
--- a/drivers/gpu/drm/i915/gt/selftest_execlists.c
+++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c
@@ -2325,13 +2325,16 @@ static int __cancel_fail(struct live_preempt_cancel *arg)
 	del_timer_sync(&engine->execlists.preempt);
 	intel_engine_flush_scheduler(engine);
 
+	engine->props.preempt_timeout_ms = 0;
 	cancel_reset_timeout(engine);
 
-	/* after failure, require heartbeats to reset device */
+	/* after failure, require fast heartbeats to reset device */
 	intel_engine_set_heartbeat(engine, 1);
 	err = wait_for_reset(engine, rq, HZ / 2);
 	intel_engine_set_heartbeat(engine,
 				   engine->defaults.heartbeat_interval_ms);
+
+	engine->props.preempt_timeout_ms = engine->defaults.preempt_timeout_ms;
 	if (err) {
 		pr_err("Cancelled inflight0 request did not reset\n");
 		goto out;
-- 
2.20.1

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply related	[flat|nested] 21+ messages in thread

* Re: [Intel-gfx] [PATCH v5] drm/i915/gt: Ratelimit heartbeat completion probing
  2021-02-11 23:14 ` [Intel-gfx] [PATCH v5] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
@ 2021-02-11 23:54   ` Tang, CQ
  0 siblings, 0 replies; 21+ messages in thread
From: Tang, CQ @ 2021-02-11 23:54 UTC (permalink / raw)
  To: Chris Wilson, intel-gfx

Do you have the patch that can apply to DII?

--CQ

> -----Original Message-----
> From: Chris Wilson <chris@chris-wilson.co.uk>
> Sent: Thursday, February 11, 2021 3:14 PM
> To: intel-gfx@lists.freedesktop.org
> Cc: Chris Wilson <chris@chris-wilson.co.uk>; Tang, CQ <cq.tang@intel.com>
> Subject: [PATCH v5] drm/i915/gt: Ratelimit heartbeat completion probing
> 
> The heartbeat runs through a few phases that we expect to complete within
> a certain number of heartbeat intervals. First we must submit the heartbeat
> to the queue, and if the queue is occupied it may take a couple of intervals
> before the heartbeat preempts the workload and is submitted to HW. Once
> running on HW, completion is not instantaneous as it may have to first reset
> the current workload before it itself runs through the empty request and
> signals completion. As such, we know that the heartbeat must take at least
> the preempt reset timeout and before we have had a chance to reset the
> engine, we do not want to issue a global reset ourselves (simply so that we
> only try to do one reset at a time and not confuse ourselves by resetting
> twice and hitting an innocent.)
> 
> So by taking into consideration that once running the request must take a
> finite amount of time, we can delay the final completion check to
> accommodate that and avoid checking too early (before we've had a chance
> to handle any engine resets required).
> 
> v3: Now with more tracking for selftests and detection of false/unexpected
> hang reports.
> 
> Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/2853
> Suggested-by: CQ Tang <cq.tang@intel.com>
> Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
> ---
>  .../gpu/drm/i915/gt/intel_engine_heartbeat.c  | 92 +++++++++++++++---
> drivers/gpu/drm/i915/gt/intel_engine_types.h  |  7 ++
>  .../drm/i915/gt/selftest_engine_heartbeat.c   | 93 ++++++++++++-------
>  drivers/gpu/drm/i915/gt/selftest_execlists.c  |  5 +-
>  4 files changed, 148 insertions(+), 49 deletions(-)
> 
> diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
> b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
> index 0b062fad1837..e14f9eab5779 100644
> --- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
> +++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
> @@ -20,6 +20,18 @@
>   * issue a reset -- in the hope that restores progress.
>   */
> 
> +#define HEARTBEAT_COMPLETION 50u /* milliseconds */
> +
> +static long completion_timeout(const struct intel_engine_cs *engine) {
> +	long timeout = HEARTBEAT_COMPLETION;
> +
> +	if (intel_engine_has_preempt_reset(engine))
> +		timeout += READ_ONCE(engine-
> >props.preempt_timeout_ms);
> +
> +	return msecs_to_jiffies(timeout);
> +}
> +
>  static bool next_heartbeat(struct intel_engine_cs *engine)  {
>  	long delay;
> @@ -29,6 +41,26 @@ static bool next_heartbeat(struct intel_engine_cs
> *engine)
>  		return false;
> 
>  	delay = msecs_to_jiffies_timeout(delay);
> +
> +	/*
> +	 * Once we submit a heartbeat to the HW, we know that it will take
> +	 * at least a certain amount of time to complete. On a hanging system
> +	 * it will first have to wait for the preempt reset timeout, and
> +	 * then it will take some time for the reset to resume with the
> +	 * heartbeat and for it to complete. So once we have submitted the
> +	 * heartbeat to HW, we can wait a while longer before declaring the
> +	 * engine stuck and forcing a reset ourselves. If we do a reset
> +	 * and the engine is also doing a reset, it is possible that we
> +	 * reset the engine twice, harming an innocent.
> +	 *
> +	 * Before we have sumitted the heartbeat, we do not want to
> change
> +	 * the interval as we want to promote the heartbeat and trigger
> +	 * preemption in a deterministic time frame.
> +	 */
> +	if (engine->heartbeat.systole &&
> +	    i915_request_is_active(engine->heartbeat.systole))
> +		delay = max(delay, completion_timeout(engine));
> +
>  	if (delay >= HZ)
>  		delay = round_jiffies_up_relative(delay);
>  	mod_delayed_work(system_highpri_wq, &engine->heartbeat.work,
> delay + 1); @@ -48,12 +80,39 @@ heartbeat_create(struct intel_context *ce,
> gfp_t gfp)
>  	return rq;
>  }
> 
> +static void
> +untrack_heartbeat(struct intel_engine_cs *engine) {
> +	struct i915_request *rq;
> +
> +	rq = engine->heartbeat.systole;
> +	if (!rq)
> +		return;
> +
> +	ENGINE_TRACE(engine, "heartbeat " RQ_FMT " completed\n",
> RQ_ARG(rq));
> +	I915_SELFTEST_ONLY(engine->heartbeat.completed++);
> +
> +	WRITE_ONCE(engine->heartbeat.systole, NULL);
> +	i915_request_put(rq);
> +}
> +
> +static void
> +track_heartbeat(struct intel_engine_cs *engine, struct i915_request
> +*rq) {
> +	ENGINE_TRACE(engine, "heartbeat " RQ_FMT " started\n",
> RQ_ARG(rq));
> +	I915_SELFTEST_ONLY(engine->heartbeat.submitted++);
> +
> +	WRITE_ONCE(engine->heartbeat.systole, i915_request_get(rq));
> +	if (!next_heartbeat(engine))
> +		untrack_heartbeat(engine);
> +}
> +
>  static void idle_pulse(struct intel_engine_cs *engine, struct i915_request
> *rq)  {
>  	engine->wakeref_serial = READ_ONCE(engine->serial) + 1;
>  	i915_request_add_active_barriers(rq);
>  	if (!engine->heartbeat.systole &&
> intel_engine_has_heartbeat(engine))
> -		engine->heartbeat.systole = i915_request_get(rq);
> +		track_heartbeat(engine, rq);
>  }
> 
>  static void heartbeat_commit(struct i915_request *rq, @@ -106,13 +165,8
> @@ static void heartbeat(struct work_struct *wrk)
>  	intel_engine_flush_scheduler(engine);
> 
>  	rq = engine->heartbeat.systole;
> -	if (rq && i915_request_completed(rq)) {
> -		ENGINE_TRACE(engine,
> -			     "heartbeat " RQ_FMT "completed\n",
> -			     RQ_ARG(rq));
> -		i915_request_put(rq);
> -		engine->heartbeat.systole = NULL;
> -	}
> +	if (rq && i915_request_completed(rq))
> +		untrack_heartbeat(engine);
> 
>  	if (!intel_engine_pm_get_if_awake(engine))
>  		return;
> @@ -120,9 +174,15 @@ static void heartbeat(struct work_struct *wrk)
>  	if (intel_gt_is_wedged(engine->gt))
>  		goto out;
> 
> +	I915_SELFTEST_ONLY(engine->heartbeat.ticks++);
> +
>  	if (engine->heartbeat.systole) {
>  		long delay = READ_ONCE(engine-
> >props.heartbeat_interval_ms);
> 
> +		if (rq->sched.attr.priority == I915_PRIORITY_BARRIER &&
> +		    i915_request_is_active(rq))
> +			delay += completion_timeout(engine);
> +
>  		/* Safeguard against too-fast worker invocations */
>  		if (!time_after(jiffies,
>  				rq->emitted_jiffies +
> msecs_to_jiffies(delay))) @@ -152,8 +212,10 @@ static void heartbeat(struct
> work_struct *wrk)
>  			attr.priority = 0;
>  			if (rq->sched.attr.priority >= attr.priority)
>  				attr.priority = I915_PRIORITY_HEARTBEAT;
> -			if (rq->sched.attr.priority >= attr.priority)
> +			if (rq->sched.attr.priority >= attr.priority) {
> +				I915_SELFTEST_ONLY(engine-
> >heartbeat.barrier++);
>  				attr.priority = I915_PRIORITY_BARRIER;
> +			}
> 
>  			ENGINE_TRACE(engine,
>  				     "bumping heartbeat " RQ_FMT "
> prio:%d\n", @@ -166,6 +228,7 @@ static void heartbeat(struct work_struct
> *wrk)
>  			ENGINE_TRACE(engine,
>  				     "heartbeat " RQ_FMT " stuck\n",
>  				     RQ_ARG(rq));
> +			I915_SELFTEST_ONLY(engine->heartbeat.reset++);
> 
>  			if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
>  				show_heartbeat(rq, engine);
> @@ -180,6 +243,11 @@ static void heartbeat(struct work_struct *wrk)
>  		goto out;
>  	}
> 
> +	/* Just completed one heartbeat, wait a tick before the next */
> +	if (rq)
> +		goto out;
> +
> +	/* The engine is parking. We can rest until the next user */
>  	serial = READ_ONCE(engine->serial);
>  	if (engine->wakeref_serial == serial)
>  		goto out;
> @@ -198,14 +266,14 @@ static void heartbeat(struct work_struct *wrk)
>  	if (IS_ERR(rq))
>  		goto unlock;
> 
> -	ENGINE_TRACE(engine, "heartbeat " RQ_FMT "started\n",
> RQ_ARG(rq));
>  	heartbeat_commit(rq, &attr);
> 
>  unlock:
>  	mutex_unlock(&ce->timeline->mutex);
>  out:
> +	intel_engine_flush_scheduler(engine);
>  	if (!engine->i915->params.enable_hangcheck
> || !next_heartbeat(engine))
> -		i915_request_put(fetch_and_zero(&engine-
> >heartbeat.systole));
> +		untrack_heartbeat(engine);
>  	intel_engine_pm_put(engine);
>  }
> 
> @@ -220,7 +288,7 @@ void intel_engine_unpark_heartbeat(struct
> intel_engine_cs *engine)  void intel_engine_park_heartbeat(struct
> intel_engine_cs *engine)  {
>  	if (cancel_delayed_work(&engine->heartbeat.work))
> -		i915_request_put(fetch_and_zero(&engine-
> >heartbeat.systole));
> +		untrack_heartbeat(engine);
>  }
> 
>  void intel_engine_init_heartbeat(struct intel_engine_cs *engine) diff --git
> a/drivers/gpu/drm/i915/gt/intel_engine_types.h
> b/drivers/gpu/drm/i915/gt/intel_engine_types.h
> index 7efa6290cc3e..1e151d550bb5 100644
> --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
> +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
> @@ -323,6 +323,13 @@ struct intel_engine_cs {
>  		struct delayed_work work;
>  		struct i915_request *systole;
>  		unsigned long blocked;
> +		I915_SELFTEST_DECLARE(struct {
> +			unsigned long submitted;
> +			unsigned long completed;
> +			unsigned long barrier;
> +			unsigned long reset;
> +			unsigned long ticks;
> +		};)
>  	} heartbeat;
> 
>  	unsigned long serial;
> diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
> b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
> index b2c369317bf1..b74459561fda 100644
> --- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
> +++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
> @@ -202,47 +202,58 @@ static int cmp_u32(const void *_a, const void *_b)
> 
>  static int __live_heartbeat_fast(struct intel_engine_cs *engine)  {
> -	const unsigned int error_threshold = max(20000u,
> jiffies_to_usecs(6));
> -	struct intel_context *ce;
> -	struct i915_request *rq;
> -	ktime_t t0, t1;
> +	const unsigned int error_threshold =
> +		max_t(unsigned int,
> +		      3 * completion_timeout(engine) * 1000,
> +		      jiffies_to_usecs(6));
> +	struct intel_context *ce = engine->kernel_context;
> +	unsigned long resets;
>  	u32 times[5];
>  	int err;
>  	int i;
> 
> -	ce = intel_context_create(engine);
> -	if (IS_ERR(ce))
> -		return PTR_ERR(ce);
> -
>  	intel_engine_pm_get(engine);
> 
>  	err = intel_engine_set_heartbeat(engine, 1);
>  	if (err)
>  		goto err_pm;
> 
> +	flush_delayed_work(&engine->heartbeat.work);
> +	while (engine->heartbeat.systole)
> +		intel_engine_park_heartbeat(engine);
> +
> +	resets = engine->heartbeat.reset;
>  	for (i = 0; i < ARRAY_SIZE(times); i++) {
> -		do {
> -			/* Manufacture a tick */
> -			intel_engine_park_heartbeat(engine);
> -			GEM_BUG_ON(engine->heartbeat.systole);
> -			engine->serial++; /*  pretend we are not idle! */
> -			intel_engine_unpark_heartbeat(engine);
> +		struct i915_sched_attr attr = { .priority =
> I915_PRIORITY_MIN };
> +		unsigned long submits = engine->heartbeat.submitted;
> +		unsigned long completed = engine->heartbeat.completed;
> +		struct i915_request *rq;
> +		ktime_t t0, t1;
> 
> -			flush_delayed_work(&engine->heartbeat.work);
> -			if (!delayed_work_pending(&engine-
> >heartbeat.work)) {
> -				pr_err("%s: heartbeat %d did not start\n",
> -				       engine->name, i);
> -				err = -EINVAL;
> -				goto err_pm;
> -			}
> +		GEM_BUG_ON(READ_ONCE(engine->heartbeat.systole));
> 
> -			rcu_read_lock();
> -			rq = READ_ONCE(engine->heartbeat.systole);
> -			if (rq)
> -				rq = i915_request_get_rcu(rq);
> -			rcu_read_unlock();
> -		} while (!rq);
> +		/* Manufacture a tick */
> +		mutex_lock(&ce->timeline->mutex);
> +		rq = heartbeat_create(ce, GFP_KERNEL);
> +		if (!IS_ERR(rq)) {
> +			i915_request_get(rq);
> +			heartbeat_commit(rq, &attr);
> +		}
> +		mutex_unlock(&ce->timeline->mutex);
> +		if (IS_ERR(rq)) {
> +			err = PTR_ERR(rq);
> +			goto err_reset;
> +		}
> 
> +		if (engine->heartbeat.submitted != submits + 1) {
> +			GEM_TRACE_ERR("Heartbeat was not tracked!\n");
> +			GEM_TRACE_DUMP();
> +			i915_request_put(rq);
> +			err = -EIO;
> +			goto err_reset;
> +		}
> +
> +		/* Time how long before the heartbeat monitor checks */
>  		t0 = ktime_get();
>  		while (rq == READ_ONCE(engine->heartbeat.systole))
>  			yield(); /* work is on the local cpu! */ @@ -250,6
> +261,20 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
> 
>  		i915_request_put(rq);
>  		times[i] = ktime_us_delta(t1, t0);
> +
> +		if (engine->heartbeat.reset != resets) {
> +			GEM_TRACE_ERR("False reset from a fast
> heartbeat!\n");
> +			GEM_TRACE_DUMP();
> +			err = -EIO;
> +			goto err_reset;
> +		}
> +
> +		if (engine->heartbeat.completed != completed + 1) {
> +			GEM_TRACE_ERR("Completed heartbeat was not
> recorded!\n");
> +			GEM_TRACE_DUMP();
> +			err = -EIO;
> +			goto err_reset;
> +		}
>  	}
> 
>  	sort(times, ARRAY_SIZE(times), sizeof(times[0]), cmp_u32, NULL);
> @@ -275,10 +300,10 @@ static int __live_heartbeat_fast(struct
> intel_engine_cs *engine)
>  		err = -EINVAL;
>  	}
> 
> +err_reset:
>  	reset_heartbeat(engine);
>  err_pm:
>  	intel_engine_pm_put(engine);
> -	intel_context_put(ce);
>  	return err;
>  }
> 
> @@ -308,20 +333,16 @@ static int __live_heartbeat_off(struct
> intel_engine_cs *engine)
> 
>  	intel_engine_pm_get(engine);
> 
> +	/* Kick once, so that we change an active heartbeat */
>  	engine->serial++;
> -	flush_delayed_work(&engine->heartbeat.work);
> -	if (!delayed_work_pending(&engine->heartbeat.work)) {
> -		pr_err("%s: heartbeat not running\n",
> -		       engine->name);
> -		err = -EINVAL;
> -		goto err_pm;
> -	}
> +	intel_engine_unpark_heartbeat(engine);
> 
>  	err = intel_engine_set_heartbeat(engine, 0);
>  	if (err)
>  		goto err_pm;
> 
> -	engine->serial++;
> +	/* The next heartbeat work should cancel the heartbeat */
> +	engine->serial++; /* pretend the engine is still active */
>  	flush_delayed_work(&engine->heartbeat.work);
>  	if (delayed_work_pending(&engine->heartbeat.work)) {
>  		pr_err("%s: heartbeat still running\n", diff --git
> a/drivers/gpu/drm/i915/gt/selftest_execlists.c
> b/drivers/gpu/drm/i915/gt/selftest_execlists.c
> index 5c8a6ccdf024..660240550097 100644
> --- a/drivers/gpu/drm/i915/gt/selftest_execlists.c
> +++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c
> @@ -2325,13 +2325,16 @@ static int __cancel_fail(struct
> live_preempt_cancel *arg)
>  	del_timer_sync(&engine->execlists.preempt);
>  	intel_engine_flush_scheduler(engine);
> 
> +	engine->props.preempt_timeout_ms = 0;
>  	cancel_reset_timeout(engine);
> 
> -	/* after failure, require heartbeats to reset device */
> +	/* after failure, require fast heartbeats to reset device */
>  	intel_engine_set_heartbeat(engine, 1);
>  	err = wait_for_reset(engine, rq, HZ / 2);
>  	intel_engine_set_heartbeat(engine,
>  				   engine->defaults.heartbeat_interval_ms);
> +
> +	engine->props.preempt_timeout_ms =
> +engine->defaults.preempt_timeout_ms;
>  	if (err) {
>  		pr_err("Cancelled inflight0 request did not reset\n");
>  		goto out;
> --
> 2.20.1

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply	[flat|nested] 21+ messages in thread

* [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev10)
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (10 preceding siblings ...)
  2021-02-11 23:14 ` [Intel-gfx] [PATCH v5] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
@ 2021-02-12  2:58 ` Patchwork
  2021-02-12  5:26 ` [Intel-gfx] ✗ Fi.CI.IGT: failure " Patchwork
                   ` (3 subsequent siblings)
  15 siblings, 0 replies; 21+ messages in thread
From: Patchwork @ 2021-02-12  2:58 UTC (permalink / raw)
  To: Chris Wilson; +Cc: intel-gfx


[-- Attachment #1.1: Type: text/plain, Size: 2850 bytes --]

== Series Details ==

Series: drm/i915/gt: Ratelimit heartbeat completion probing (rev10)
URL   : https://patchwork.freedesktop.org/series/86665/
State : success

== Summary ==

CI Bug Log - changes from CI_DRM_9767 -> Patchwork_19667
====================================================

Summary
-------

  **SUCCESS**

  No regressions found.

  External URL: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/index.html

Known issues
------------

  Here are the changes found in Patchwork_19667 that come from known issues:

### CI changes ###


### IGT changes ###

#### Issues hit ####

  * igt@kms_chamelium@hdmi-hpd-fast:
    - fi-kbl-7500u:       [PASS][1] -> [DMESG-FAIL][2] ([i915#165])
   [1]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/fi-kbl-7500u/igt@kms_chamelium@hdmi-hpd-fast.html
   [2]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/fi-kbl-7500u/igt@kms_chamelium@hdmi-hpd-fast.html

  * igt@prime_self_import@basic-with_one_bo_two_files:
    - fi-tgl-y:           [PASS][3] -> [DMESG-WARN][4] ([i915#402]) +1 similar issue
   [3]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/fi-tgl-y/igt@prime_self_import@basic-with_one_bo_two_files.html
   [4]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/fi-tgl-y/igt@prime_self_import@basic-with_one_bo_two_files.html

  
#### Possible fixes ####

  * igt@debugfs_test@read_all_entries:
    - fi-tgl-y:           [DMESG-WARN][5] ([i915#402]) -> [PASS][6] +2 similar issues
   [5]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/fi-tgl-y/igt@debugfs_test@read_all_entries.html
   [6]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/fi-tgl-y/igt@debugfs_test@read_all_entries.html

  
  {name}: This element is suppressed. This means it is ignored when computing
          the status of the difference (SUCCESS, WARNING, or FAILURE).

  [i915#165]: https://gitlab.freedesktop.org/drm/intel/issues/165
  [i915#2992]: https://gitlab.freedesktop.org/drm/intel/issues/2992
  [i915#402]: https://gitlab.freedesktop.org/drm/intel/issues/402


Participating hosts (43 -> 38)
------------------------------

  Additional (1): fi-ehl-2 
  Missing    (6): fi-ilk-m540 fi-hsw-4200u fi-byt-j1900 fi-bsw-cyan fi-ctg-p8600 fi-bdw-samus 


Build changes
-------------

  * Linux: CI_DRM_9767 -> Patchwork_19667

  CI-20190529: 20190529
  CI_DRM_9767: 975f60d1f2dc8e653110d33517f00515dede35bb @ git://anongit.freedesktop.org/gfx-ci/linux
  IGT_6001: d0d6f5e14ef181c93e4b503b05d9c18fa480e09d @ git://anongit.freedesktop.org/xorg/app/intel-gpu-tools
  Patchwork_19667: 1c2fc38a94ff6e886d2fe58f3e1446fa58ab4a19 @ git://anongit.freedesktop.org/gfx-ci/linux


== Linux commits ==

1c2fc38a94ff drm/i915/gt: Ratelimit heartbeat completion probing

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/index.html

[-- Attachment #1.2: Type: text/html, Size: 3522 bytes --]

[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply	[flat|nested] 21+ messages in thread

* [Intel-gfx] ✗ Fi.CI.IGT: failure for drm/i915/gt: Ratelimit heartbeat completion probing (rev10)
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (11 preceding siblings ...)
  2021-02-12  2:58 ` [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev10) Patchwork
@ 2021-02-12  5:26 ` Patchwork
  2021-02-12 12:58 ` [Intel-gfx] [PATCH v6] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (2 subsequent siblings)
  15 siblings, 0 replies; 21+ messages in thread
From: Patchwork @ 2021-02-12  5:26 UTC (permalink / raw)
  To: Chris Wilson; +Cc: intel-gfx


[-- Attachment #1.1: Type: text/plain, Size: 30282 bytes --]

== Series Details ==

Series: drm/i915/gt: Ratelimit heartbeat completion probing (rev10)
URL   : https://patchwork.freedesktop.org/series/86665/
State : failure

== Summary ==

CI Bug Log - changes from CI_DRM_9767_full -> Patchwork_19667_full
====================================================

Summary
-------

  **FAILURE**

  Serious unknown changes coming with Patchwork_19667_full absolutely need to be
  verified manually.
  
  If you think the reported changes have nothing to do with the changes
  introduced in Patchwork_19667_full, please notify your bug team to allow them
  to document this new failure mode, which will reduce false positives in CI.

  

Possible new issues
-------------------

  Here are the unknown changes that may have been introduced in Patchwork_19667_full:

### IGT changes ###

#### Possible regressions ####

  * igt@sysfs_heartbeat_interval@nopreempt@vcs0:
    - shard-skl:          [PASS][1] -> [FAIL][2] +1 similar issue
   [1]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl7/igt@sysfs_heartbeat_interval@nopreempt@vcs0.html
   [2]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl9/igt@sysfs_heartbeat_interval@nopreempt@vcs0.html
    - shard-glk:          [PASS][3] -> [FAIL][4] +1 similar issue
   [3]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-glk5/igt@sysfs_heartbeat_interval@nopreempt@vcs0.html
   [4]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-glk5/igt@sysfs_heartbeat_interval@nopreempt@vcs0.html
    - shard-apl:          [PASS][5] -> [FAIL][6] +1 similar issue
   [5]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-apl4/igt@sysfs_heartbeat_interval@nopreempt@vcs0.html
   [6]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-apl8/igt@sysfs_heartbeat_interval@nopreempt@vcs0.html

  * igt@sysfs_heartbeat_interval@nopreempt@vecs0:
    - shard-iclb:         [PASS][7] -> [FAIL][8] +1 similar issue
   [7]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-iclb8/igt@sysfs_heartbeat_interval@nopreempt@vecs0.html
   [8]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb3/igt@sysfs_heartbeat_interval@nopreempt@vecs0.html
    - shard-kbl:          NOTRUN -> [FAIL][9] +2 similar issues
   [9]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-kbl7/igt@sysfs_heartbeat_interval@nopreempt@vecs0.html

  

### Piglit changes ###

#### Possible regressions ####

  * spec@glsl-4.00@execution@built-in-functions@fs-op-mult-dmat2x3-dmat4x2 (NEW):
    - {pig-icl-1065g7}:   NOTRUN -> [CRASH][10]
   [10]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/pig-icl-1065g7/spec@glsl-4.00@execution@built-in-functions@fs-op-mult-dmat2x3-dmat4x2.html

  * spec@glsl-4.00@execution@built-in-functions@gs-transpose-dmat4 (NEW):
    - {pig-icl-1065g7}:   NOTRUN -> [INCOMPLETE][11]
   [11]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/pig-icl-1065g7/spec@glsl-4.00@execution@built-in-functions@gs-transpose-dmat4.html

  
New tests
---------

  New tests have been introduced between CI_DRM_9767_full and Patchwork_19667_full:

### New Piglit tests (2) ###

  * spec@glsl-4.00@execution@built-in-functions@fs-op-mult-dmat2x3-dmat4x2:
    - Statuses : 1 crash(s)
    - Exec time: [32.45] s

  * spec@glsl-4.00@execution@built-in-functions@gs-transpose-dmat4:
    - Statuses : 1 incomplete(s)
    - Exec time: [0.0] s

  

Known issues
------------

  Here are the changes found in Patchwork_19667_full that come from known issues:

### IGT changes ###

#### Issues hit ####

  * igt@gem_exec_fair@basic-deadline:
    - shard-tglb:         [PASS][12] -> [FAIL][13] ([i915#2846])
   [12]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-tglb7/igt@gem_exec_fair@basic-deadline.html
   [13]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-tglb2/igt@gem_exec_fair@basic-deadline.html

  * igt@gem_exec_fair@basic-none-vip@rcs0:
    - shard-glk:          [PASS][14] -> [FAIL][15] ([i915#2842])
   [14]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-glk5/igt@gem_exec_fair@basic-none-vip@rcs0.html
   [15]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-glk6/igt@gem_exec_fair@basic-none-vip@rcs0.html

  * igt@gem_exec_fair@basic-pace@rcs0:
    - shard-kbl:          [PASS][16] -> [SKIP][17] ([fdo#109271])
   [16]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-kbl7/igt@gem_exec_fair@basic-pace@rcs0.html
   [17]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-kbl4/igt@gem_exec_fair@basic-pace@rcs0.html

  * igt@gem_exec_fair@basic-pace@vcs1:
    - shard-iclb:         NOTRUN -> [FAIL][18] ([i915#2842])
   [18]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb4/igt@gem_exec_fair@basic-pace@vcs1.html

  * igt@gem_exec_fair@basic-pace@vecs0:
    - shard-kbl:          [PASS][19] -> [FAIL][20] ([i915#2842])
   [19]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-kbl7/igt@gem_exec_fair@basic-pace@vecs0.html
   [20]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-kbl4/igt@gem_exec_fair@basic-pace@vecs0.html

  * igt@gem_exec_params@no-vebox:
    - shard-iclb:         NOTRUN -> [SKIP][21] ([fdo#109283])
   [21]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb5/igt@gem_exec_params@no-vebox.html

  * igt@gem_exec_reloc@basic-wide-active@vcs1:
    - shard-iclb:         NOTRUN -> [FAIL][22] ([i915#2389])
   [22]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb4/igt@gem_exec_reloc@basic-wide-active@vcs1.html

  * igt@gem_exec_whisper@basic-fds-forked-all:
    - shard-iclb:         [PASS][23] -> [INCOMPLETE][24] ([i915#1895])
   [23]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-iclb2/igt@gem_exec_whisper@basic-fds-forked-all.html
   [24]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb7/igt@gem_exec_whisper@basic-fds-forked-all.html

  * igt@gem_userptr_blits@process-exit-mmap@wb:
    - shard-apl:          NOTRUN -> [SKIP][25] ([fdo#109271] / [i915#1699]) +3 similar issues
   [25]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-apl7/igt@gem_userptr_blits@process-exit-mmap@wb.html
    - shard-skl:          NOTRUN -> [SKIP][26] ([fdo#109271] / [i915#1699]) +3 similar issues
   [26]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl9/igt@gem_userptr_blits@process-exit-mmap@wb.html

  * igt@gem_userptr_blits@readonly-pwrite-unsync:
    - shard-iclb:         NOTRUN -> [SKIP][27] ([fdo#110426] / [i915#1704])
   [27]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb5/igt@gem_userptr_blits@readonly-pwrite-unsync.html

  * igt@i915_pm_dc@dc6-psr:
    - shard-iclb:         [PASS][28] -> [FAIL][29] ([i915#454])
   [28]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-iclb7/igt@i915_pm_dc@dc6-psr.html
   [29]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb4/igt@i915_pm_dc@dc6-psr.html

  * igt@i915_selftest@live@hangcheck:
    - shard-hsw:          [PASS][30] -> [INCOMPLETE][31] ([i915#2782])
   [30]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-hsw5/igt@i915_selftest@live@hangcheck.html
   [31]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-hsw4/igt@i915_selftest@live@hangcheck.html

  * igt@i915_suspend@forcewake:
    - shard-skl:          [PASS][32] -> [INCOMPLETE][33] ([i915#636])
   [32]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl6/igt@i915_suspend@forcewake.html
   [33]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl6/igt@i915_suspend@forcewake.html

  * igt@i915_suspend@sysfs-reader:
    - shard-apl:          [PASS][34] -> [DMESG-WARN][35] ([i915#180]) +4 similar issues
   [34]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-apl6/igt@i915_suspend@sysfs-reader.html
   [35]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-apl1/igt@i915_suspend@sysfs-reader.html

  * igt@kms_ccs@pipe-c-bad-aux-stride:
    - shard-skl:          NOTRUN -> [SKIP][36] ([fdo#109271] / [fdo#111304])
   [36]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl7/igt@kms_ccs@pipe-c-bad-aux-stride.html

  * igt@kms_chamelium@dp-hpd-for-each-pipe:
    - shard-apl:          NOTRUN -> [SKIP][37] ([fdo#109271] / [fdo#111827]) +6 similar issues
   [37]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-apl7/igt@kms_chamelium@dp-hpd-for-each-pipe.html

  * igt@kms_chamelium@hdmi-hpd-fast:
    - shard-iclb:         NOTRUN -> [SKIP][38] ([fdo#109284] / [fdo#111827]) +1 similar issue
   [38]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb5/igt@kms_chamelium@hdmi-hpd-fast.html

  * igt@kms_chamelium@hdmi-hpd-with-enabled-mode:
    - shard-kbl:          NOTRUN -> [SKIP][39] ([fdo#109271] / [fdo#111827]) +2 similar issues
   [39]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-kbl7/igt@kms_chamelium@hdmi-hpd-with-enabled-mode.html

  * igt@kms_chamelium@vga-hpd-after-suspend:
    - shard-skl:          NOTRUN -> [SKIP][40] ([fdo#109271] / [fdo#111827]) +7 similar issues
   [40]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl7/igt@kms_chamelium@vga-hpd-after-suspend.html

  * igt@kms_color@pipe-a-ctm-0-75:
    - shard-iclb:         NOTRUN -> [FAIL][41] ([i915#1149] / [i915#315])
   [41]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb5/igt@kms_color@pipe-a-ctm-0-75.html

  * igt@kms_color@pipe-d-ctm-blue-to-red:
    - shard-iclb:         NOTRUN -> [SKIP][42] ([fdo#109278] / [i915#1149])
   [42]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb5/igt@kms_color@pipe-d-ctm-blue-to-red.html

  * igt@kms_content_protection@atomic-dpms:
    - shard-apl:          NOTRUN -> [TIMEOUT][43] ([i915#1319])
   [43]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-apl7/igt@kms_content_protection@atomic-dpms.html

  * igt@kms_content_protection@legacy:
    - shard-iclb:         NOTRUN -> [SKIP][44] ([fdo#109300] / [fdo#111066])
   [44]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb5/igt@kms_content_protection@legacy.html

  * igt@kms_cursor_crc@pipe-c-cursor-64x21-offscreen:
    - shard-skl:          [PASS][45] -> [FAIL][46] ([i915#54]) +6 similar issues
   [45]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl2/igt@kms_cursor_crc@pipe-c-cursor-64x21-offscreen.html
   [46]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl2/igt@kms_cursor_crc@pipe-c-cursor-64x21-offscreen.html

  * igt@kms_cursor_legacy@cursorb-vs-flipa-atomic-transitions-varying-size:
    - shard-iclb:         NOTRUN -> [SKIP][47] ([fdo#109274] / [fdo#109278])
   [47]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb5/igt@kms_cursor_legacy@cursorb-vs-flipa-atomic-transitions-varying-size.html

  * igt@kms_cursor_legacy@flip-vs-cursor-atomic-transitions:
    - shard-skl:          NOTRUN -> [FAIL][48] ([i915#2346]) +1 similar issue
   [48]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl9/igt@kms_cursor_legacy@flip-vs-cursor-atomic-transitions.html

  * igt@kms_cursor_legacy@pipe-d-torture-move:
    - shard-skl:          NOTRUN -> [SKIP][49] ([fdo#109271]) +80 similar issues
   [49]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl9/igt@kms_cursor_legacy@pipe-d-torture-move.html

  * igt@kms_draw_crc@draw-method-rgb565-blt-ytiled:
    - shard-skl:          [PASS][50] -> [DMESG-WARN][51] ([i915#1982])
   [50]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl9/igt@kms_draw_crc@draw-method-rgb565-blt-ytiled.html
   [51]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl1/igt@kms_draw_crc@draw-method-rgb565-blt-ytiled.html

  * igt@kms_flip@flip-vs-expired-vblank@a-edp1:
    - shard-skl:          NOTRUN -> [FAIL][52] ([i915#79])
   [52]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl7/igt@kms_flip@flip-vs-expired-vblank@a-edp1.html

  * igt@kms_flip@plain-flip-fb-recreate@b-edp1:
    - shard-skl:          [PASS][53] -> [FAIL][54] ([i915#2122]) +1 similar issue
   [53]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl5/igt@kms_flip@plain-flip-fb-recreate@b-edp1.html
   [54]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl3/igt@kms_flip@plain-flip-fb-recreate@b-edp1.html

  * igt@kms_flip@plain-flip-ts-check@b-edp1:
    - shard-skl:          NOTRUN -> [FAIL][55] ([i915#2122])
   [55]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl7/igt@kms_flip@plain-flip-ts-check@b-edp1.html

  * igt@kms_flip_scaled_crc@flip-32bpp-ytile-to-32bpp-ytileccs:
    - shard-skl:          NOTRUN -> [FAIL][56] ([i915#2628])
   [56]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl1/igt@kms_flip_scaled_crc@flip-32bpp-ytile-to-32bpp-ytileccs.html

  * igt@kms_frontbuffer_tracking@fbc-2p-primscrn-pri-shrfb-draw-mmap-cpu:
    - shard-apl:          NOTRUN -> [SKIP][57] ([fdo#109271]) +68 similar issues
   [57]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-apl7/igt@kms_frontbuffer_tracking@fbc-2p-primscrn-pri-shrfb-draw-mmap-cpu.html

  * igt@kms_frontbuffer_tracking@psr-2p-primscrn-pri-shrfb-draw-mmap-gtt:
    - shard-iclb:         NOTRUN -> [SKIP][58] ([fdo#109280]) +5 similar issues
   [58]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb5/igt@kms_frontbuffer_tracking@psr-2p-primscrn-pri-shrfb-draw-mmap-gtt.html

  * igt@kms_hdr@bpc-switch-suspend:
    - shard-skl:          NOTRUN -> [FAIL][59] ([i915#1188])
   [59]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl9/igt@kms_hdr@bpc-switch-suspend.html

  * igt@kms_plane_alpha_blend@pipe-b-alpha-opaque-fb:
    - shard-kbl:          NOTRUN -> [FAIL][60] ([fdo#108145] / [i915#265])
   [60]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-kbl7/igt@kms_plane_alpha_blend@pipe-b-alpha-opaque-fb.html

  * igt@kms_plane_alpha_blend@pipe-b-alpha-transparent-fb:
    - shard-apl:          NOTRUN -> [FAIL][61] ([i915#265])
   [61]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-apl7/igt@kms_plane_alpha_blend@pipe-b-alpha-transparent-fb.html
    - shard-skl:          NOTRUN -> [FAIL][62] ([i915#265])
   [62]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl9/igt@kms_plane_alpha_blend@pipe-b-alpha-transparent-fb.html

  * igt@kms_plane_alpha_blend@pipe-c-constant-alpha-min:
    - shard-skl:          NOTRUN -> [FAIL][63] ([fdo#108145] / [i915#265]) +1 similar issue
   [63]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl7/igt@kms_plane_alpha_blend@pipe-c-constant-alpha-min.html

  * igt@kms_plane_alpha_blend@pipe-c-coverage-7efc:
    - shard-skl:          [PASS][64] -> [FAIL][65] ([fdo#108145] / [i915#265])
   [64]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl2/igt@kms_plane_alpha_blend@pipe-c-coverage-7efc.html
   [65]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl2/igt@kms_plane_alpha_blend@pipe-c-coverage-7efc.html

  * igt@kms_plane_alpha_blend@pipe-d-constant-alpha-max:
    - shard-iclb:         NOTRUN -> [SKIP][66] ([fdo#109278]) +1 similar issue
   [66]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb5/igt@kms_plane_alpha_blend@pipe-d-constant-alpha-max.html

  * igt@kms_psr2_sf@overlay-primary-update-sf-dmg-area-3:
    - shard-apl:          NOTRUN -> [SKIP][67] ([fdo#109271] / [i915#658]) +2 similar issues
   [67]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-apl7/igt@kms_psr2_sf@overlay-primary-update-sf-dmg-area-3.html
    - shard-skl:          NOTRUN -> [SKIP][68] ([fdo#109271] / [i915#658]) +2 similar issues
   [68]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl7/igt@kms_psr2_sf@overlay-primary-update-sf-dmg-area-3.html

  * igt@kms_psr2_sf@overlay-primary-update-sf-dmg-area-4:
    - shard-iclb:         NOTRUN -> [SKIP][69] ([i915#658])
   [69]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb5/igt@kms_psr2_sf@overlay-primary-update-sf-dmg-area-4.html

  * igt@kms_psr2_sf@plane-move-sf-dmg-area-0:
    - shard-kbl:          NOTRUN -> [SKIP][70] ([fdo#109271] / [i915#658])
   [70]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-kbl7/igt@kms_psr2_sf@plane-move-sf-dmg-area-0.html

  * igt@kms_psr@psr2_cursor_plane_move:
    - shard-iclb:         [PASS][71] -> [SKIP][72] ([fdo#109441]) +1 similar issue
   [71]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-iclb2/igt@kms_psr@psr2_cursor_plane_move.html
   [72]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb8/igt@kms_psr@psr2_cursor_plane_move.html

  * igt@kms_psr@psr2_primary_render:
    - shard-iclb:         NOTRUN -> [SKIP][73] ([fdo#109441])
   [73]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb5/igt@kms_psr@psr2_primary_render.html

  * igt@kms_writeback@writeback-pixel-formats:
    - shard-iclb:         NOTRUN -> [SKIP][74] ([i915#2437])
   [74]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb5/igt@kms_writeback@writeback-pixel-formats.html

  * igt@nouveau_crc@pipe-b-source-outp-complete:
    - shard-iclb:         NOTRUN -> [SKIP][75] ([i915#2530])
   [75]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb5/igt@nouveau_crc@pipe-b-source-outp-complete.html

  * igt@perf@polling-parameterized:
    - shard-skl:          NOTRUN -> [FAIL][76] ([i915#1542])
   [76]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl7/igt@perf@polling-parameterized.html

  * igt@prime_nv_api@i915_nv_double_import:
    - shard-kbl:          NOTRUN -> [SKIP][77] ([fdo#109271]) +37 similar issues
   [77]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-kbl7/igt@prime_nv_api@i915_nv_double_import.html

  * igt@sysfs_clients@busy@vcs0:
    - shard-skl:          [PASS][78] -> [FAIL][79] ([i915#3018])
   [78]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl5/igt@sysfs_clients@busy@vcs0.html
   [79]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl1/igt@sysfs_clients@busy@vcs0.html

  * igt@sysfs_clients@busy@vecs0:
    - shard-kbl:          [PASS][80] -> [FAIL][81] ([i915#3009])
   [80]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-kbl3/igt@sysfs_clients@busy@vecs0.html
   [81]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-kbl3/igt@sysfs_clients@busy@vecs0.html

  * igt@sysfs_clients@split-10@bcs0:
    - shard-apl:          NOTRUN -> [SKIP][82] ([fdo#109271] / [i915#3026])
   [82]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-apl7/igt@sysfs_clients@split-10@bcs0.html

  * igt@sysfs_clients@split-10@rcs0:
    - shard-skl:          NOTRUN -> [SKIP][83] ([fdo#109271] / [i915#3026]) +2 similar issues
   [83]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl7/igt@sysfs_clients@split-10@rcs0.html

  * igt@sysfs_heartbeat_interval@nopreempt@bcs0:
    - shard-apl:          [PASS][84] -> [FAIL][85] ([i915#1778]) +1 similar issue
   [84]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-apl4/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html
   [85]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-apl8/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html
    - shard-iclb:         [PASS][86] -> [FAIL][87] ([i915#1778]) +1 similar issue
   [86]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-iclb8/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html
   [87]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb3/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html
    - shard-glk:          [PASS][88] -> [FAIL][89] ([i915#1778]) +1 similar issue
   [88]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-glk5/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html
   [89]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-glk5/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html
    - shard-kbl:          NOTRUN -> [FAIL][90] ([i915#1778]) +1 similar issue
   [90]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-kbl7/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html

  * igt@sysfs_heartbeat_interval@nopreempt@rcs0:
    - shard-skl:          [PASS][91] -> [FAIL][92] ([i915#1778]) +1 similar issue
   [91]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl7/igt@sysfs_heartbeat_interval@nopreempt@rcs0.html
   [92]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl9/igt@sysfs_heartbeat_interval@nopreempt@rcs0.html

  * igt@sysfs_heartbeat_interval@nopreempt@vcs1:
    - shard-tglb:         [PASS][93] -> [FAIL][94] ([i915#1778]) +4 similar issues
   [93]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-tglb8/igt@sysfs_heartbeat_interval@nopreempt@vcs1.html
   [94]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-tglb5/igt@sysfs_heartbeat_interval@nopreempt@vcs1.html

  
#### Possible fixes ####

  * igt@drm_import_export@prime:
    - shard-kbl:          [INCOMPLETE][95] ([i915#2895] / [i915#2944]) -> [PASS][96]
   [95]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-kbl6/igt@drm_import_export@prime.html
   [96]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-kbl7/igt@drm_import_export@prime.html

  * igt@gem_eio@unwedge-stress:
    - shard-skl:          [TIMEOUT][97] ([i915#1037] / [i915#2771]) -> [PASS][98]
   [97]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl4/igt@gem_eio@unwedge-stress.html
   [98]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl2/igt@gem_eio@unwedge-stress.html

  * igt@gem_exec_balancer@hang:
    - shard-iclb:         [INCOMPLETE][99] ([i915#1895] / [i915#2295] / [i915#3031]) -> [PASS][100]
   [99]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-iclb4/igt@gem_exec_balancer@hang.html
   [100]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb5/igt@gem_exec_balancer@hang.html

  * igt@gem_exec_fair@basic-none@vcs0:
    - shard-apl:          [FAIL][101] ([i915#2842]) -> [PASS][102]
   [101]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-apl4/igt@gem_exec_fair@basic-none@vcs0.html
   [102]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-apl8/igt@gem_exec_fair@basic-none@vcs0.html

  * igt@gem_exec_fair@basic-pace-share@rcs0:
    - shard-glk:          [FAIL][103] ([i915#2842]) -> [PASS][104] +1 similar issue
   [103]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-glk9/igt@gem_exec_fair@basic-pace-share@rcs0.html
   [104]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-glk7/igt@gem_exec_fair@basic-pace-share@rcs0.html

  * igt@gem_exec_fair@basic-pace@rcs0:
    - shard-tglb:         [FAIL][105] ([i915#2842]) -> [PASS][106]
   [105]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-tglb8/igt@gem_exec_fair@basic-pace@rcs0.html
   [106]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-tglb6/igt@gem_exec_fair@basic-pace@rcs0.html

  * igt@gem_exec_fair@basic-pace@vcs1:
    - shard-kbl:          [SKIP][107] ([fdo#109271]) -> [PASS][108]
   [107]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-kbl7/igt@gem_exec_fair@basic-pace@vcs1.html
   [108]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-kbl4/igt@gem_exec_fair@basic-pace@vcs1.html

  * igt@gem_exec_reloc@basic-many-active@rcs0:
    - shard-hsw:          [FAIL][109] ([i915#2389]) -> [PASS][110]
   [109]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-hsw1/igt@gem_exec_reloc@basic-many-active@rcs0.html
   [110]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-hsw8/igt@gem_exec_reloc@basic-many-active@rcs0.html

  * igt@gem_exec_schedule@u-fairslice@vcs0:
    - shard-skl:          [DMESG-WARN][111] ([i915#1610] / [i915#2803]) -> [PASS][112]
   [111]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl5/igt@gem_exec_schedule@u-fairslice@vcs0.html
   [112]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl1/igt@gem_exec_schedule@u-fairslice@vcs0.html

  * igt@i915_selftest@live@client:
    - shard-glk:          [DMESG-FAIL][113] ([i915#3047]) -> [PASS][114]
   [113]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-glk7/igt@i915_selftest@live@client.html
   [114]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-glk5/igt@i915_selftest@live@client.html

  * igt@kms_color@pipe-b-ctm-0-5:
    - shard-skl:          [DMESG-WARN][115] ([i915#1982]) -> [PASS][116]
   [115]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl10/igt@kms_color@pipe-b-ctm-0-5.html
   [116]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl4/igt@kms_color@pipe-b-ctm-0-5.html

  * igt@kms_cursor_crc@pipe-b-cursor-64x21-offscreen:
    - shard-skl:          [FAIL][117] ([i915#54]) -> [PASS][118] +5 similar issues
   [117]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl6/igt@kms_cursor_crc@pipe-b-cursor-64x21-offscreen.html
   [118]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl6/igt@kms_cursor_crc@pipe-b-cursor-64x21-offscreen.html

  * igt@kms_cursor_crc@pipe-b-cursor-suspend:
    - shard-skl:          [INCOMPLETE][119] ([i915#2405] / [i915#300]) -> [PASS][120]
   [119]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl2/igt@kms_cursor_crc@pipe-b-cursor-suspend.html
   [120]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl9/igt@kms_cursor_crc@pipe-b-cursor-suspend.html

  * igt@kms_flip@flip-vs-suspend-interruptible@c-dp1:
    - shard-apl:          [DMESG-WARN][121] ([i915#180]) -> [PASS][122] +2 similar issues
   [121]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-apl6/igt@kms_flip@flip-vs-suspend-interruptible@c-dp1.html
   [122]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-apl3/igt@kms_flip@flip-vs-suspend-interruptible@c-dp1.html

  * igt@kms_plane_alpha_blend@pipe-b-coverage-7efc:
    - shard-skl:          [FAIL][123] ([fdo#108145] / [i915#265]) -> [PASS][124]
   [123]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl8/igt@kms_plane_alpha_blend@pipe-b-coverage-7efc.html
   [124]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl3/igt@kms_plane_alpha_blend@pipe-b-coverage-7efc.html

  * igt@kms_psr@psr2_sprite_render:
    - shard-iclb:         [SKIP][125] ([fdo#109441]) -> [PASS][126] +1 similar issue
   [125]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-iclb6/igt@kms_psr@psr2_sprite_render.html
   [126]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb2/igt@kms_psr@psr2_sprite_render.html

  * igt@sysfs_clients@create:
    - shard-glk:          [FAIL][127] -> [PASS][128]
   [127]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-glk5/igt@sysfs_clients@create.html
   [128]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-glk7/igt@sysfs_clients@create.html

  * igt@sysfs_clients@recycle:
    - shard-tglb:         [FAIL][129] ([i915#3028]) -> [PASS][130]
   [129]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-tglb5/igt@sysfs_clients@recycle.html
   [130]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-tglb2/igt@sysfs_clients@recycle.html

  * {igt@sysfs_clients@recycle-many}:
    - shard-apl:          [FAIL][131] ([i915#3028]) -> [PASS][132]
   [131]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-apl7/igt@sysfs_clients@recycle-many.html
   [132]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-apl1/igt@sysfs_clients@recycle-many.html
    - shard-skl:          [FAIL][133] ([i915#3028]) -> [PASS][134]
   [133]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl5/igt@sysfs_clients@recycle-many.html
   [134]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl1/igt@sysfs_clients@recycle-many.html

  * igt@sysfs_clients@split-25@vecs0:
    - shard-skl:          [SKIP][135] ([fdo#109271]) -> [PASS][136]
   [135]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-skl8/igt@sysfs_clients@split-25@vecs0.html
   [136]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-skl3/igt@sysfs_clients@split-25@vecs0.html

  
#### Warnings ####

  * igt@gem_exec_fair@basic-none-rrul@rcs0:
    - shard-iclb:         [FAIL][137] ([i915#2852]) -> [FAIL][138] ([i915#2842])
   [137]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-iclb6/igt@gem_exec_fair@basic-none-rrul@rcs0.html
   [138]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb2/igt@gem_exec_fair@basic-none-rrul@rcs0.html

  * igt@gem_exec_fair@basic-pace@vcs0:
    - shard-kbl:          [FAIL][139] ([i915#2842]) -> [SKIP][140] ([fdo#109271])
   [139]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-kbl7/igt@gem_exec_fair@basic-pace@vcs0.html
   [140]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-kbl4/igt@gem_exec_fair@basic-pace@vcs0.html

  * igt@i915_pm_rc6_residency@rc6-fence:
    - shard-iclb:         [WARN][141] ([i915#2681] / [i915#2684]) -> [WARN][142] ([i915#1804] / [i915#2684]) +1 similar issue
   [141]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-iclb8/igt@i915_pm_rc6_residency@rc6-fence.html
   [142]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb3/igt@i915_pm_rc6_residency@rc6-fence.html

  * igt@kms_psr2_sf@plane-move-sf-dmg-area-3:
    - shard-iclb:         [SKIP][143] ([i915#658]) -> [SKIP][144] ([i915#2920]) +1 similar issue
   [143]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-iclb6/igt@kms_psr2_sf@plane-move-sf-dmg-area-3.html
   [144]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/shard-iclb2/igt@kms_psr2_sf@plane-move-sf-dmg-area-3.html

  * igt@kms_psr2_sf@primary-plane-update-sf-dmg-area-2:
    - shard-iclb:         [SKIP][145] ([i915#2920]) -> [SKIP][146] ([i915#658]) +1 similar issue
   [145]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9767/shard-iclb2/igt@kms_psr2_sf@primary-plane-update-sf-dmg-area-2.html
   [146]: https://intel-gfx-ci

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19667/index.html

[-- Attachment #1.2: Type: text/html, Size: 33468 bytes --]

[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply	[flat|nested] 21+ messages in thread

* [Intel-gfx] [PATCH v6] drm/i915/gt: Ratelimit heartbeat completion probing
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (12 preceding siblings ...)
  2021-02-12  5:26 ` [Intel-gfx] ✗ Fi.CI.IGT: failure " Patchwork
@ 2021-02-12 12:58 ` Chris Wilson
  2021-02-12 14:00 ` [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev11) Patchwork
  2021-02-12 15:08 ` [Intel-gfx] ✗ Fi.CI.IGT: failure " Patchwork
  15 siblings, 0 replies; 21+ messages in thread
From: Chris Wilson @ 2021-02-12 12:58 UTC (permalink / raw)
  To: intel-gfx; +Cc: Chris Wilson

The heartbeat runs through a few phases that we expect to complete
within a certain number of heartbeat intervals. First we must submit the
heartbeat to the queue, and if the queue is occupied it may take a
couple of intervals before the heartbeat preempts the workload and is
submitted to HW. Once running on HW, completion is not instantaneous as
it may have to first reset the current workload before it itself runs
through the empty request and signals completion. As such, we know that
the heartbeat must take at least the preempt reset timeout and before we
have had a chance to reset the engine, we do not want to issue a global
reset ourselves (simply so that we only try to do one reset at a time
and not confuse ourselves by resetting twice and hitting an innocent.)

So by taking into consideration that once running the request must take
a finite amount of time, we can delay the final completion check to
accommodate that and avoid checking too early (before we've had a chance
to handle any engine resets required).

v3: Now with more tracking for selftests and detection of
false/unexpected hang reports.

v2,4,5: Add a completion handler for the heartbeat to speed up discovery
of a successful heartbeat. Remove it, but then put it back to handle
large mismatches between the sysfs properties and reality.

Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/2853
Suggested-by: CQ Tang <cq.tang@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
 .../gpu/drm/i915/gt/intel_engine_heartbeat.c  | 107 +++++++++++++++---
 drivers/gpu/drm/i915/gt/intel_engine_types.h  |   8 ++
 .../drm/i915/gt/selftest_engine_heartbeat.c   | 103 +++++++++++------
 drivers/gpu/drm/i915/gt/selftest_execlists.c  |   5 +-
 4 files changed, 173 insertions(+), 50 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
index 0b062fad1837..7e7d2ff2299b 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
@@ -20,6 +20,18 @@
  * issue a reset -- in the hope that restores progress.
  */
 
+#define HEARTBEAT_COMPLETION 50u /* milliseconds */
+
+static long completion_timeout(const struct intel_engine_cs *engine)
+{
+	long timeout = HEARTBEAT_COMPLETION;
+
+	if (intel_engine_has_preempt_reset(engine))
+		timeout += READ_ONCE(engine->props.preempt_timeout_ms);
+
+	return msecs_to_jiffies(timeout);
+}
+
 static bool next_heartbeat(struct intel_engine_cs *engine)
 {
 	long delay;
@@ -29,6 +41,26 @@ static bool next_heartbeat(struct intel_engine_cs *engine)
 		return false;
 
 	delay = msecs_to_jiffies_timeout(delay);
+
+	/*
+	 * Once we submit a heartbeat to the HW, we know that it will take
+	 * at least a certain amount of time to complete. On a hanging system
+	 * it will first have to wait for the preempt reset timeout, and
+	 * then it will take some time for the reset to resume with the
+	 * heartbeat and for it to complete. So once we have submitted the
+	 * heartbeat to HW, we can wait a while longer before declaring the
+	 * engine stuck and forcing a reset ourselves. If we do a reset
+	 * and the engine is also doing a reset, it is possible that we
+	 * reset the engine twice, harming an innocent.
+	 *
+	 * Before we have sumitted the heartbeat, we do not want to change
+	 * the interval as we want to promote the heartbeat and trigger
+	 * preemption in a deterministic time frame.
+	 */
+	if (engine->heartbeat.systole &&
+	    i915_request_is_active(engine->heartbeat.systole))
+		delay = max(delay, completion_timeout(engine));
+
 	if (delay >= HZ)
 		delay = round_jiffies_up_relative(delay);
 	mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, delay + 1);
@@ -48,12 +80,52 @@ heartbeat_create(struct intel_context *ce, gfp_t gfp)
 	return rq;
 }
 
+static void
+untrack_heartbeat(struct intel_engine_cs *engine)
+{
+	struct i915_request *rq;
+
+	rq = engine->heartbeat.systole;
+	if (!rq)
+		return;
+
+	ENGINE_TRACE(engine, "heartbeat " RQ_FMT " completed\n", RQ_ARG(rq));
+	I915_SELFTEST_ONLY(engine->heartbeat.completed++);
+
+	dma_fence_remove_callback(&rq->fence, &engine->heartbeat.cb);
+	WRITE_ONCE(engine->heartbeat.systole, NULL);
+
+	i915_request_put(rq);
+}
+
+static void defibrillator(struct dma_fence *f, struct dma_fence_cb *cb)
+{
+	struct intel_engine_cs *engine =
+		container_of(cb, typeof(*engine), heartbeat.cb);
+
+	if (READ_ONCE(engine->heartbeat.systole))
+		mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, 0);
+}
+
+static void
+track_heartbeat(struct intel_engine_cs *engine, struct i915_request *rq)
+{
+	ENGINE_TRACE(engine, "heartbeat " RQ_FMT " started\n", RQ_ARG(rq));
+	I915_SELFTEST_ONLY(engine->heartbeat.submitted++);
+
+	if (!dma_fence_add_callback(&rq->fence, &engine->heartbeat.cb, defibrillator))
+		WRITE_ONCE(engine->heartbeat.systole, i915_request_get(rq));
+
+	if (!next_heartbeat(engine))
+		untrack_heartbeat(engine);
+}
+
 static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq)
 {
 	engine->wakeref_serial = READ_ONCE(engine->serial) + 1;
 	i915_request_add_active_barriers(rq);
 	if (!engine->heartbeat.systole && intel_engine_has_heartbeat(engine))
-		engine->heartbeat.systole = i915_request_get(rq);
+		track_heartbeat(engine, rq);
 }
 
 static void heartbeat_commit(struct i915_request *rq,
@@ -106,13 +178,8 @@ static void heartbeat(struct work_struct *wrk)
 	intel_engine_flush_scheduler(engine);
 
 	rq = engine->heartbeat.systole;
-	if (rq && i915_request_completed(rq)) {
-		ENGINE_TRACE(engine,
-			     "heartbeat " RQ_FMT "completed\n",
-			     RQ_ARG(rq));
-		i915_request_put(rq);
-		engine->heartbeat.systole = NULL;
-	}
+	if (rq && i915_request_completed(rq))
+		untrack_heartbeat(engine);
 
 	if (!intel_engine_pm_get_if_awake(engine))
 		return;
@@ -120,9 +187,15 @@ static void heartbeat(struct work_struct *wrk)
 	if (intel_gt_is_wedged(engine->gt))
 		goto out;
 
+	I915_SELFTEST_ONLY(engine->heartbeat.ticks++);
+
 	if (engine->heartbeat.systole) {
 		long delay = READ_ONCE(engine->props.heartbeat_interval_ms);
 
+		if (rq->sched.attr.priority == I915_PRIORITY_BARRIER &&
+		    i915_request_is_active(rq))
+			delay += completion_timeout(engine);
+
 		/* Safeguard against too-fast worker invocations */
 		if (!time_after(jiffies,
 				rq->emitted_jiffies + msecs_to_jiffies(delay)))
@@ -152,8 +225,10 @@ static void heartbeat(struct work_struct *wrk)
 			attr.priority = 0;
 			if (rq->sched.attr.priority >= attr.priority)
 				attr.priority = I915_PRIORITY_HEARTBEAT;
-			if (rq->sched.attr.priority >= attr.priority)
+			if (rq->sched.attr.priority >= attr.priority) {
+				I915_SELFTEST_ONLY(engine->heartbeat.barrier++);
 				attr.priority = I915_PRIORITY_BARRIER;
+			}
 
 			ENGINE_TRACE(engine,
 				     "bumping heartbeat " RQ_FMT " prio:%d\n",
@@ -166,6 +241,7 @@ static void heartbeat(struct work_struct *wrk)
 			ENGINE_TRACE(engine,
 				     "heartbeat " RQ_FMT " stuck\n",
 				     RQ_ARG(rq));
+			I915_SELFTEST_ONLY(engine->heartbeat.reset++);
 
 			if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 				show_heartbeat(rq, engine);
@@ -180,6 +256,11 @@ static void heartbeat(struct work_struct *wrk)
 		goto out;
 	}
 
+	/* Just completed one heartbeat, wait a tick before the next */
+	if (rq)
+		goto out;
+
+	/* The engine is parking. We can rest until the next user */
 	serial = READ_ONCE(engine->serial);
 	if (engine->wakeref_serial == serial)
 		goto out;
@@ -198,14 +279,14 @@ static void heartbeat(struct work_struct *wrk)
 	if (IS_ERR(rq))
 		goto unlock;
 
-	ENGINE_TRACE(engine, "heartbeat " RQ_FMT "started\n", RQ_ARG(rq));
 	heartbeat_commit(rq, &attr);
 
 unlock:
 	mutex_unlock(&ce->timeline->mutex);
 out:
+	intel_engine_flush_scheduler(engine);
 	if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+		untrack_heartbeat(engine);
 	intel_engine_pm_put(engine);
 }
 
@@ -219,8 +300,8 @@ void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine)
 
 void intel_engine_park_heartbeat(struct intel_engine_cs *engine)
 {
-	if (cancel_delayed_work(&engine->heartbeat.work))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+	while (cancel_delayed_work(&engine->heartbeat.work))
+		untrack_heartbeat(engine);
 }
 
 void intel_engine_init_heartbeat(struct intel_engine_cs *engine)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 7efa6290cc3e..5ad8d26e2fa0 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -322,7 +322,15 @@ struct intel_engine_cs {
 	struct {
 		struct delayed_work work;
 		struct i915_request *systole;
+		struct dma_fence_cb cb;
 		unsigned long blocked;
+		I915_SELFTEST_DECLARE(struct {
+			unsigned long submitted;
+			unsigned long completed;
+			unsigned long barrier;
+			unsigned long reset;
+			unsigned long ticks;
+		};)
 	} heartbeat;
 
 	unsigned long serial;
diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
index b2c369317bf1..e20a73ad9968 100644
--- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
@@ -202,47 +202,58 @@ static int cmp_u32(const void *_a, const void *_b)
 
 static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 {
-	const unsigned int error_threshold = max(20000u, jiffies_to_usecs(6));
-	struct intel_context *ce;
-	struct i915_request *rq;
-	ktime_t t0, t1;
+	const unsigned int error_threshold =
+		max_t(unsigned int,
+		      3 * completion_timeout(engine) * 1000,
+		      jiffies_to_usecs(6));
+	struct intel_context *ce = engine->kernel_context;
+	unsigned long resets;
 	u32 times[5];
 	int err;
 	int i;
 
-	ce = intel_context_create(engine);
-	if (IS_ERR(ce))
-		return PTR_ERR(ce);
-
 	intel_engine_pm_get(engine);
 
 	err = intel_engine_set_heartbeat(engine, 1);
 	if (err)
 		goto err_pm;
 
+	flush_delayed_work(&engine->heartbeat.work);
+	while (engine->heartbeat.systole)
+		intel_engine_park_heartbeat(engine);
+
+	resets = engine->heartbeat.reset;
 	for (i = 0; i < ARRAY_SIZE(times); i++) {
-		do {
-			/* Manufacture a tick */
-			intel_engine_park_heartbeat(engine);
-			GEM_BUG_ON(engine->heartbeat.systole);
-			engine->serial++; /*  pretend we are not idle! */
-			intel_engine_unpark_heartbeat(engine);
+		struct i915_sched_attr attr = { .priority = I915_PRIORITY_MIN };
+		unsigned long submits = engine->heartbeat.submitted;
+		unsigned long completed = engine->heartbeat.completed;
+		struct i915_request *rq;
+		ktime_t t0, t1;
 
-			flush_delayed_work(&engine->heartbeat.work);
-			if (!delayed_work_pending(&engine->heartbeat.work)) {
-				pr_err("%s: heartbeat %d did not start\n",
-				       engine->name, i);
-				err = -EINVAL;
-				goto err_pm;
-			}
+		GEM_BUG_ON(READ_ONCE(engine->heartbeat.systole));
 
-			rcu_read_lock();
-			rq = READ_ONCE(engine->heartbeat.systole);
-			if (rq)
-				rq = i915_request_get_rcu(rq);
-			rcu_read_unlock();
-		} while (!rq);
+		/* Manufacture a tick */
+		mutex_lock(&ce->timeline->mutex);
+		rq = heartbeat_create(ce, GFP_KERNEL);
+		if (!IS_ERR(rq)) {
+			i915_request_get(rq);
+			heartbeat_commit(rq, &attr);
+		}
+		mutex_unlock(&ce->timeline->mutex);
+		if (IS_ERR(rq)) {
+			err = PTR_ERR(rq);
+			goto err_reset;
+		}
 
+		if (engine->heartbeat.submitted != submits + 1) {
+			GEM_TRACE_ERR("Heartbeat was not tracked!\n");
+			GEM_TRACE_DUMP();
+			i915_request_put(rq);
+			err = -EIO;
+			goto err_reset;
+		}
+
+		/* Time how long before the heartbeat monitor checks */
 		t0 = ktime_get();
 		while (rq == READ_ONCE(engine->heartbeat.systole))
 			yield(); /* work is on the local cpu! */
@@ -250,6 +261,20 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 
 		i915_request_put(rq);
 		times[i] = ktime_us_delta(t1, t0);
+
+		if (engine->heartbeat.reset != resets) {
+			GEM_TRACE_ERR("False reset from a fast heartbeat!\n");
+			GEM_TRACE_DUMP();
+			err = -EIO;
+			goto err_reset;
+		}
+
+		if (engine->heartbeat.completed != completed + 1) {
+			GEM_TRACE_ERR("Completed heartbeat was not recorded!\n");
+			GEM_TRACE_DUMP();
+			err = -EIO;
+			goto err_reset;
+		}
 	}
 
 	sort(times, ARRAY_SIZE(times), sizeof(times[0]), cmp_u32, NULL);
@@ -275,10 +300,10 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 		err = -EINVAL;
 	}
 
+err_reset:
 	reset_heartbeat(engine);
 err_pm:
 	intel_engine_pm_put(engine);
-	intel_context_put(ce);
 	return err;
 }
 
@@ -304,24 +329,23 @@ static int live_heartbeat_fast(void *arg)
 
 static int __live_heartbeat_off(struct intel_engine_cs *engine)
 {
+	unsigned long submits;
 	int err;
 
 	intel_engine_pm_get(engine);
 
+	/* Kick once, so that we change an active heartbeat */
 	engine->serial++;
-	flush_delayed_work(&engine->heartbeat.work);
-	if (!delayed_work_pending(&engine->heartbeat.work)) {
-		pr_err("%s: heartbeat not running\n",
-		       engine->name);
-		err = -EINVAL;
-		goto err_pm;
-	}
+	intel_engine_unpark_heartbeat(engine);
 
 	err = intel_engine_set_heartbeat(engine, 0);
 	if (err)
 		goto err_pm;
 
-	engine->serial++;
+	submits = READ_ONCE(engine->heartbeat.submitted);
+
+	/* The next heartbeat work should cancel the heartbeat */
+	engine->serial++; /* pretend the engine is still active */
 	flush_delayed_work(&engine->heartbeat.work);
 	if (delayed_work_pending(&engine->heartbeat.work)) {
 		pr_err("%s: heartbeat still running\n",
@@ -337,6 +361,13 @@ static int __live_heartbeat_off(struct intel_engine_cs *engine)
 		goto err_beat;
 	}
 
+	if (READ_ONCE(engine->heartbeat.submitted) != submits) {
+		pr_err("%s: heartbeat was submitted after being turned off\n",
+		       engine->name);
+		err = -EINVAL;
+		goto err_beat;
+	}
+
 err_beat:
 	reset_heartbeat(engine);
 err_pm:
diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c
index 5c8a6ccdf024..660240550097 100644
--- a/drivers/gpu/drm/i915/gt/selftest_execlists.c
+++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c
@@ -2325,13 +2325,16 @@ static int __cancel_fail(struct live_preempt_cancel *arg)
 	del_timer_sync(&engine->execlists.preempt);
 	intel_engine_flush_scheduler(engine);
 
+	engine->props.preempt_timeout_ms = 0;
 	cancel_reset_timeout(engine);
 
-	/* after failure, require heartbeats to reset device */
+	/* after failure, require fast heartbeats to reset device */
 	intel_engine_set_heartbeat(engine, 1);
 	err = wait_for_reset(engine, rq, HZ / 2);
 	intel_engine_set_heartbeat(engine,
 				   engine->defaults.heartbeat_interval_ms);
+
+	engine->props.preempt_timeout_ms = engine->defaults.preempt_timeout_ms;
 	if (err) {
 		pr_err("Cancelled inflight0 request did not reset\n");
 		goto out;
-- 
2.20.1

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply related	[flat|nested] 21+ messages in thread

* [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev11)
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (13 preceding siblings ...)
  2021-02-12 12:58 ` [Intel-gfx] [PATCH v6] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
@ 2021-02-12 14:00 ` Patchwork
  2021-02-12 15:08 ` [Intel-gfx] ✗ Fi.CI.IGT: failure " Patchwork
  15 siblings, 0 replies; 21+ messages in thread
From: Patchwork @ 2021-02-12 14:00 UTC (permalink / raw)
  To: Chris Wilson; +Cc: intel-gfx


[-- Attachment #1.1: Type: text/plain, Size: 6896 bytes --]

== Series Details ==

Series: drm/i915/gt: Ratelimit heartbeat completion probing (rev11)
URL   : https://patchwork.freedesktop.org/series/86665/
State : success

== Summary ==

CI Bug Log - changes from CI_DRM_9769 -> Patchwork_19669
====================================================

Summary
-------

  **SUCCESS**

  No regressions found.

  External URL: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/index.html

Possible new issues
-------------------

  Here are the unknown changes that may have been introduced in Patchwork_19669:

### IGT changes ###

#### Suppressed ####

  The following results come from untrusted machines, tests, or statuses.
  They do not affect the overall result.

  * igt@i915_module_load@reload:
    - {fi-ehl-2}:         NOTRUN -> [DMESG-WARN][1] +33 similar issues
   [1]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-ehl-2/igt@i915_module_load@reload.html

  * igt@i915_selftest@live@gt_engines:
    - {fi-ehl-2}:         NOTRUN -> [DMESG-FAIL][2]
   [2]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-ehl-2/igt@i915_selftest@live@gt_engines.html

  * igt@i915_selftest@live@hangcheck:
    - {fi-ehl-2}:         NOTRUN -> [INCOMPLETE][3]
   [3]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-ehl-2/igt@i915_selftest@live@hangcheck.html

  * igt@kms_force_connector_basic@force-load-detect:
    - {fi-ehl-2}:         NOTRUN -> [SKIP][4] +2 similar issues
   [4]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-ehl-2/igt@kms_force_connector_basic@force-load-detect.html

  * igt@runner@aborted:
    - {fi-ehl-2}:         NOTRUN -> [FAIL][5]
   [5]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-ehl-2/igt@runner@aborted.html

  
Known issues
------------

  Here are the changes found in Patchwork_19669 that come from known issues:

### CI changes ###

#### Possible fixes ####

  * boot:
    - {fi-ehl-2}:         [FAIL][6] ([i915#2992]) -> [PASS][7]
   [6]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/fi-ehl-2/boot.html
   [7]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-ehl-2/boot.html

  

### IGT changes ###

#### Issues hit ####

  * igt@amdgpu/amd_basic@cs-compute:
    - fi-cfl-guc:         NOTRUN -> [SKIP][8] ([fdo#109271]) +25 similar issues
   [8]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-cfl-guc/igt@amdgpu/amd_basic@cs-compute.html

  * igt@amdgpu/amd_basic@query-info:
    - fi-bsw-kefka:       NOTRUN -> [SKIP][9] ([fdo#109271]) +17 similar issues
   [9]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-bsw-kefka/igt@amdgpu/amd_basic@query-info.html

  * igt@amdgpu/amd_basic@userptr:
    - fi-byt-j1900:       NOTRUN -> [SKIP][10] ([fdo#109271]) +17 similar issues
   [10]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-byt-j1900/igt@amdgpu/amd_basic@userptr.html

  * igt@i915_selftest@live@hangcheck:
    - fi-tgl-u2:          [PASS][11] -> [INCOMPLETE][12] ([i915#750])
   [11]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/fi-tgl-u2/igt@i915_selftest@live@hangcheck.html
   [12]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-tgl-u2/igt@i915_selftest@live@hangcheck.html

  * igt@kms_chamelium@dp-edid-read:
    - fi-cfl-guc:         NOTRUN -> [SKIP][13] ([fdo#109271] / [fdo#111827]) +8 similar issues
   [13]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-cfl-guc/igt@kms_chamelium@dp-edid-read.html

  * igt@kms_pipe_crc_basic@compare-crc-sanitycheck-pipe-d:
    - fi-cfl-guc:         NOTRUN -> [SKIP][14] ([fdo#109271] / [i915#533])
   [14]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-cfl-guc/igt@kms_pipe_crc_basic@compare-crc-sanitycheck-pipe-d.html

  * igt@prime_self_import@basic-with_two_bos:
    - fi-tgl-y:           [PASS][15] -> [DMESG-WARN][16] ([i915#402])
   [15]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/fi-tgl-y/igt@prime_self_import@basic-with_two_bos.html
   [16]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-tgl-y/igt@prime_self_import@basic-with_two_bos.html

  * igt@runner@aborted:
    - fi-tgl-u2:          NOTRUN -> [FAIL][17] ([i915#2966])
   [17]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-tgl-u2/igt@runner@aborted.html

  
#### Possible fixes ####

  * igt@fbdev@read:
    - fi-tgl-y:           [DMESG-WARN][18] ([i915#402]) -> [PASS][19] +1 similar issue
   [18]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/fi-tgl-y/igt@fbdev@read.html
   [19]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-tgl-y/igt@fbdev@read.html

  * igt@i915_pm_rpm@module-reload:
    - fi-byt-j1900:       [INCOMPLETE][20] ([i915#142] / [i915#2405]) -> [PASS][21]
   [20]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/fi-byt-j1900/igt@i915_pm_rpm@module-reload.html
   [21]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-byt-j1900/igt@i915_pm_rpm@module-reload.html

  * igt@i915_selftest@live@execlists:
    - fi-bsw-kefka:       [INCOMPLETE][22] ([i915#2940]) -> [PASS][23]
   [22]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/fi-bsw-kefka/igt@i915_selftest@live@execlists.html
   [23]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/fi-bsw-kefka/igt@i915_selftest@live@execlists.html

  
  {name}: This element is suppressed. This means it is ignored when computing
          the status of the difference (SUCCESS, WARNING, or FAILURE).

  [fdo#109271]: https://bugs.freedesktop.org/show_bug.cgi?id=109271
  [fdo#111827]: https://bugs.freedesktop.org/show_bug.cgi?id=111827
  [i915#142]: https://gitlab.freedesktop.org/drm/intel/issues/142
  [i915#2405]: https://gitlab.freedesktop.org/drm/intel/issues/2405
  [i915#2940]: https://gitlab.freedesktop.org/drm/intel/issues/2940
  [i915#2966]: https://gitlab.freedesktop.org/drm/intel/issues/2966
  [i915#2992]: https://gitlab.freedesktop.org/drm/intel/issues/2992
  [i915#402]: https://gitlab.freedesktop.org/drm/intel/issues/402
  [i915#533]: https://gitlab.freedesktop.org/drm/intel/issues/533
  [i915#750]: https://gitlab.freedesktop.org/drm/intel/issues/750


Participating hosts (44 -> 39)
------------------------------

  Additional (1): fi-cfl-guc 
  Missing    (6): fi-ilk-m540 fi-bdw-5557u fi-hsw-4200u fi-bsw-cyan fi-ctg-p8600 fi-bdw-samus 


Build changes
-------------

  * Linux: CI_DRM_9769 -> Patchwork_19669

  CI-20190529: 20190529
  CI_DRM_9769: 2b48b43b8f964195fc483aff9468a8b75177b08b @ git://anongit.freedesktop.org/gfx-ci/linux
  IGT_6001: d0d6f5e14ef181c93e4b503b05d9c18fa480e09d @ git://anongit.freedesktop.org/xorg/app/intel-gpu-tools
  Patchwork_19669: 4c329784c103f5a39cf333187a2a3c9afa965eeb @ git://anongit.freedesktop.org/gfx-ci/linux


== Linux commits ==

4c329784c103 drm/i915/gt: Ratelimit heartbeat completion probing

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/index.html

[-- Attachment #1.2: Type: text/html, Size: 8291 bytes --]

[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply	[flat|nested] 21+ messages in thread

* [Intel-gfx] ✗ Fi.CI.IGT: failure for drm/i915/gt: Ratelimit heartbeat completion probing (rev11)
  2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
                   ` (14 preceding siblings ...)
  2021-02-12 14:00 ` [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev11) Patchwork
@ 2021-02-12 15:08 ` Patchwork
  15 siblings, 0 replies; 21+ messages in thread
From: Patchwork @ 2021-02-12 15:08 UTC (permalink / raw)
  To: Chris Wilson; +Cc: intel-gfx


[-- Attachment #1.1: Type: text/plain, Size: 30281 bytes --]

== Series Details ==

Series: drm/i915/gt: Ratelimit heartbeat completion probing (rev11)
URL   : https://patchwork.freedesktop.org/series/86665/
State : failure

== Summary ==

CI Bug Log - changes from CI_DRM_9769_full -> Patchwork_19669_full
====================================================

Summary
-------

  **FAILURE**

  Serious unknown changes coming with Patchwork_19669_full absolutely need to be
  verified manually.
  
  If you think the reported changes have nothing to do with the changes
  introduced in Patchwork_19669_full, please notify your bug team to allow them
  to document this new failure mode, which will reduce false positives in CI.

  

Possible new issues
-------------------

  Here are the unknown changes that may have been introduced in Patchwork_19669_full:

### IGT changes ###

#### Possible regressions ####

  * igt@sysfs_heartbeat_interval@nopreempt@vcs0:
    - shard-skl:          [PASS][1] -> [FAIL][2] +1 similar issue
   [1]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-skl4/igt@sysfs_heartbeat_interval@nopreempt@vcs0.html
   [2]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-skl5/igt@sysfs_heartbeat_interval@nopreempt@vcs0.html
    - shard-glk:          [PASS][3] -> [FAIL][4] +1 similar issue
   [3]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-glk2/igt@sysfs_heartbeat_interval@nopreempt@vcs0.html
   [4]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-glk6/igt@sysfs_heartbeat_interval@nopreempt@vcs0.html
    - shard-apl:          [PASS][5] -> [FAIL][6] +1 similar issue
   [5]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-apl2/igt@sysfs_heartbeat_interval@nopreempt@vcs0.html
   [6]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-apl7/igt@sysfs_heartbeat_interval@nopreempt@vcs0.html

  * igt@sysfs_heartbeat_interval@nopreempt@vcs1:
    - shard-iclb:         NOTRUN -> [FAIL][7]
   [7]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb2/igt@sysfs_heartbeat_interval@nopreempt@vcs1.html

  * igt@sysfs_heartbeat_interval@nopreempt@vecs0:
    - shard-iclb:         [PASS][8] -> [FAIL][9] +1 similar issue
   [8]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-iclb3/igt@sysfs_heartbeat_interval@nopreempt@vecs0.html
   [9]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb2/igt@sysfs_heartbeat_interval@nopreempt@vecs0.html
    - shard-kbl:          [PASS][10] -> [FAIL][11] +2 similar issues
   [10]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-kbl3/igt@sysfs_heartbeat_interval@nopreempt@vecs0.html
   [11]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl1/igt@sysfs_heartbeat_interval@nopreempt@vecs0.html

  
Known issues
------------

  Here are the changes found in Patchwork_19669_full that come from known issues:

### IGT changes ###

#### Issues hit ####

  * igt@gem_ctx_persistence@engines-hang:
    - shard-hsw:          NOTRUN -> [SKIP][12] ([fdo#109271] / [i915#1099]) +1 similar issue
   [12]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-hsw1/igt@gem_ctx_persistence@engines-hang.html

  * igt@gem_eio@in-flight-contexts-10ms:
    - shard-hsw:          [PASS][13] -> [INCOMPLETE][14] ([i915#1037] / [i915#2870])
   [13]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-hsw5/igt@gem_eio@in-flight-contexts-10ms.html
   [14]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-hsw4/igt@gem_eio@in-flight-contexts-10ms.html

  * igt@gem_eio@unwedge-stress:
    - shard-tglb:         [PASS][15] -> [TIMEOUT][16] ([i915#1037] / [i915#3063])
   [15]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-tglb7/igt@gem_eio@unwedge-stress.html
   [16]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-tglb7/igt@gem_eio@unwedge-stress.html

  * igt@gem_exec_fair@basic-none@vcs0:
    - shard-kbl:          [PASS][17] -> [FAIL][18] ([i915#2842])
   [17]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-kbl6/igt@gem_exec_fair@basic-none@vcs0.html
   [18]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl1/igt@gem_exec_fair@basic-none@vcs0.html

  * igt@gem_exec_fair@basic-pace-share@rcs0:
    - shard-glk:          [PASS][19] -> [FAIL][20] ([i915#2842])
   [19]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-glk7/igt@gem_exec_fair@basic-pace-share@rcs0.html
   [20]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-glk3/igt@gem_exec_fair@basic-pace-share@rcs0.html

  * igt@gem_exec_fair@basic-pace@rcs0:
    - shard-kbl:          [PASS][21] -> [SKIP][22] ([fdo#109271])
   [21]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-kbl4/igt@gem_exec_fair@basic-pace@rcs0.html
   [22]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl4/igt@gem_exec_fair@basic-pace@rcs0.html

  * igt@gem_exec_fair@basic-pace@vcs1:
    - shard-iclb:         NOTRUN -> [FAIL][23] ([i915#2842]) +1 similar issue
   [23]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb1/igt@gem_exec_fair@basic-pace@vcs1.html

  * igt@gem_exec_params@no-vebox:
    - shard-iclb:         NOTRUN -> [SKIP][24] ([fdo#109283])
   [24]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb8/igt@gem_exec_params@no-vebox.html

  * igt@gem_exec_reloc@basic-wide-active@vcs1:
    - shard-iclb:         NOTRUN -> [FAIL][25] ([i915#2389]) +1 similar issue
   [25]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb1/igt@gem_exec_reloc@basic-wide-active@vcs1.html

  * igt@gem_exec_schedule@u-fairslice@vcs0:
    - shard-skl:          NOTRUN -> [DMESG-WARN][26] ([i915#1610] / [i915#2803])
   [26]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-skl3/igt@gem_exec_schedule@u-fairslice@vcs0.html

  * igt@gem_exec_whisper@basic-contexts-forked-all:
    - shard-glk:          [PASS][27] -> [DMESG-WARN][28] ([i915#118] / [i915#95])
   [27]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-glk7/igt@gem_exec_whisper@basic-contexts-forked-all.html
   [28]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-glk3/igt@gem_exec_whisper@basic-contexts-forked-all.html

  * igt@gem_exec_whisper@basic-fds-forked-all:
    - shard-hsw:          [PASS][29] -> [INCOMPLETE][30] ([i915#2055])
   [29]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-hsw5/igt@gem_exec_whisper@basic-fds-forked-all.html
   [30]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-hsw5/igt@gem_exec_whisper@basic-fds-forked-all.html

  * igt@gem_huc_copy@huc-copy:
    - shard-kbl:          NOTRUN -> [SKIP][31] ([fdo#109271] / [i915#2190])
   [31]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl7/igt@gem_huc_copy@huc-copy.html

  * igt@gem_render_copy@linear-to-vebox-yf-tiled:
    - shard-iclb:         NOTRUN -> [SKIP][32] ([i915#768])
   [32]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@gem_render_copy@linear-to-vebox-yf-tiled.html

  * igt@gem_userptr_blits@process-exit-mmap@gtt:
    - shard-kbl:          NOTRUN -> [SKIP][33] ([fdo#109271] / [i915#1699]) +3 similar issues
   [33]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl1/igt@gem_userptr_blits@process-exit-mmap@gtt.html

  * igt@gem_userptr_blits@readonly-mmap-unsync@wb:
    - shard-iclb:         NOTRUN -> [SKIP][34] ([i915#1704]) +3 similar issues
   [34]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@gem_userptr_blits@readonly-mmap-unsync@wb.html

  * igt@gem_userptr_blits@readonly-pwrite-unsync:
    - shard-iclb:         NOTRUN -> [SKIP][35] ([fdo#110426] / [i915#1704])
   [35]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb8/igt@gem_userptr_blits@readonly-pwrite-unsync.html

  * igt@gen7_exec_parse@oacontrol-tracking:
    - shard-iclb:         NOTRUN -> [SKIP][36] ([fdo#109289]) +3 similar issues
   [36]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@gen7_exec_parse@oacontrol-tracking.html

  * igt@gen9_exec_parse@batch-without-end:
    - shard-tglb:         NOTRUN -> [SKIP][37] ([fdo#112306])
   [37]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-tglb7/igt@gen9_exec_parse@batch-without-end.html

  * igt@gen9_exec_parse@bb-oversize:
    - shard-tglb:         NOTRUN -> [SKIP][38] ([i915#2527])
   [38]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-tglb7/igt@gen9_exec_parse@bb-oversize.html

  * igt@gen9_exec_parse@bb-secure:
    - shard-iclb:         NOTRUN -> [SKIP][39] ([fdo#112306])
   [39]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@gen9_exec_parse@bb-secure.html

  * igt@i915_pm_lpsp@screens-disabled:
    - shard-tglb:         NOTRUN -> [SKIP][40] ([i915#1902])
   [40]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-tglb7/igt@i915_pm_lpsp@screens-disabled.html

  * igt@i915_pm_rc6_residency@rc6-idle:
    - shard-iclb:         NOTRUN -> [WARN][41] ([i915#2684])
   [41]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@i915_pm_rc6_residency@rc6-idle.html

  * igt@i915_suspend@sysfs-reader:
    - shard-kbl:          [PASS][42] -> [DMESG-WARN][43] ([i915#180]) +1 similar issue
   [42]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-kbl7/igt@i915_suspend@sysfs-reader.html
   [43]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl6/igt@i915_suspend@sysfs-reader.html

  * igt@kms_big_fb@x-tiled-64bpp-rotate-90:
    - shard-iclb:         NOTRUN -> [SKIP][44] ([fdo#110725] / [fdo#111614])
   [44]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@kms_big_fb@x-tiled-64bpp-rotate-90.html

  * igt@kms_big_joiner@basic:
    - shard-iclb:         NOTRUN -> [SKIP][45] ([i915#2705])
   [45]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@kms_big_joiner@basic.html

  * igt@kms_chamelium@dp-hpd:
    - shard-skl:          NOTRUN -> [SKIP][46] ([fdo#109271] / [fdo#111827]) +2 similar issues
   [46]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-skl3/igt@kms_chamelium@dp-hpd.html

  * igt@kms_chamelium@hdmi-hpd-with-enabled-mode:
    - shard-hsw:          NOTRUN -> [SKIP][47] ([fdo#109271] / [fdo#111827]) +2 similar issues
   [47]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-hsw1/igt@kms_chamelium@hdmi-hpd-with-enabled-mode.html

  * igt@kms_chamelium@vga-hpd:
    - shard-apl:          NOTRUN -> [SKIP][48] ([fdo#109271] / [fdo#111827]) +2 similar issues
   [48]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-apl7/igt@kms_chamelium@vga-hpd.html

  * igt@kms_color@pipe-a-ctm-0-75:
    - shard-iclb:         NOTRUN -> [FAIL][49] ([i915#1149] / [i915#315])
   [49]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb8/igt@kms_color@pipe-a-ctm-0-75.html

  * igt@kms_color@pipe-d-ctm-max:
    - shard-iclb:         NOTRUN -> [SKIP][50] ([fdo#109278] / [i915#1149]) +1 similar issue
   [50]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@kms_color@pipe-d-ctm-max.html

  * igt@kms_color_chamelium@pipe-a-ctm-blue-to-red:
    - shard-kbl:          NOTRUN -> [SKIP][51] ([fdo#109271] / [fdo#111827]) +4 similar issues
   [51]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl1/igt@kms_color_chamelium@pipe-a-ctm-blue-to-red.html

  * igt@kms_color_chamelium@pipe-c-ctm-0-5:
    - shard-iclb:         NOTRUN -> [SKIP][52] ([fdo#109284] / [fdo#111827]) +5 similar issues
   [52]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@kms_color_chamelium@pipe-c-ctm-0-5.html

  * igt@kms_color_chamelium@pipe-d-ctm-red-to-blue:
    - shard-tglb:         NOTRUN -> [SKIP][53] ([fdo#109284] / [fdo#111827])
   [53]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-tglb7/igt@kms_color_chamelium@pipe-d-ctm-red-to-blue.html

  * igt@kms_content_protection@legacy:
    - shard-iclb:         NOTRUN -> [SKIP][54] ([fdo#109300] / [fdo#111066]) +1 similar issue
   [54]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb8/igt@kms_content_protection@legacy.html

  * igt@kms_cursor_crc@pipe-a-cursor-128x128-random:
    - shard-skl:          NOTRUN -> [FAIL][55] ([i915#54]) +1 similar issue
   [55]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-skl3/igt@kms_cursor_crc@pipe-a-cursor-128x128-random.html

  * igt@kms_cursor_crc@pipe-c-cursor-512x512-offscreen:
    - shard-iclb:         NOTRUN -> [SKIP][56] ([fdo#109278] / [fdo#109279]) +1 similar issue
   [56]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@kms_cursor_crc@pipe-c-cursor-512x512-offscreen.html

  * igt@kms_cursor_crc@pipe-c-cursor-512x512-sliding:
    - shard-tglb:         NOTRUN -> [SKIP][57] ([fdo#109279])
   [57]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-tglb7/igt@kms_cursor_crc@pipe-c-cursor-512x512-sliding.html

  * igt@kms_cursor_crc@pipe-c-cursor-64x64-random:
    - shard-skl:          [PASS][58] -> [FAIL][59] ([i915#54]) +7 similar issues
   [58]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-skl8/igt@kms_cursor_crc@pipe-c-cursor-64x64-random.html
   [59]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-skl8/igt@kms_cursor_crc@pipe-c-cursor-64x64-random.html

  * igt@kms_cursor_crc@pipe-d-cursor-256x256-random:
    - shard-hsw:          NOTRUN -> [SKIP][60] ([fdo#109271] / [i915#533]) +4 similar issues
   [60]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-hsw1/igt@kms_cursor_crc@pipe-d-cursor-256x256-random.html

  * igt@kms_cursor_legacy@cursora-vs-flipb-varying-size:
    - shard-iclb:         NOTRUN -> [SKIP][61] ([fdo#109274] / [fdo#109278]) +1 similar issue
   [61]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@kms_cursor_legacy@cursora-vs-flipb-varying-size.html

  * igt@kms_cursor_legacy@flip-vs-cursor-varying-size:
    - shard-tglb:         [PASS][62] -> [FAIL][63] ([i915#2346])
   [62]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-tglb1/igt@kms_cursor_legacy@flip-vs-cursor-varying-size.html
   [63]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-tglb8/igt@kms_cursor_legacy@flip-vs-cursor-varying-size.html

  * igt@kms_fbcon_fbt@fbc-suspend:
    - shard-iclb:         [PASS][64] -> [INCOMPLETE][65] ([i915#1185])
   [64]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-iclb1/igt@kms_fbcon_fbt@fbc-suspend.html
   [65]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb3/igt@kms_fbcon_fbt@fbc-suspend.html

  * igt@kms_flip@2x-flip-vs-panning:
    - shard-iclb:         NOTRUN -> [SKIP][66] ([fdo#109274]) +1 similar issue
   [66]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@kms_flip@2x-flip-vs-panning.html

  * igt@kms_flip@dpms-vs-vblank-race-interruptible@b-edp1:
    - shard-skl:          [PASS][67] -> [FAIL][68] ([i915#407])
   [67]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-skl10/igt@kms_flip@dpms-vs-vblank-race-interruptible@b-edp1.html
   [68]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-skl9/igt@kms_flip@dpms-vs-vblank-race-interruptible@b-edp1.html

  * igt@kms_flip@flip-vs-suspend-interruptible@a-dp1:
    - shard-apl:          [PASS][69] -> [DMESG-WARN][70] ([i915#180])
   [69]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-apl6/igt@kms_flip@flip-vs-suspend-interruptible@a-dp1.html
   [70]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-apl2/igt@kms_flip@flip-vs-suspend-interruptible@a-dp1.html

  * igt@kms_flip@flip-vs-suspend-interruptible@c-edp1:
    - shard-skl:          [PASS][71] -> [INCOMPLETE][72] ([i915#146] / [i915#198] / [i915#2295])
   [71]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-skl5/igt@kms_flip@flip-vs-suspend-interruptible@c-edp1.html
   [72]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-skl2/igt@kms_flip@flip-vs-suspend-interruptible@c-edp1.html

  * igt@kms_flip_scaled_crc@flip-32bpp-ytile-to-32bpp-ytileccs:
    - shard-kbl:          NOTRUN -> [FAIL][73] ([i915#2641])
   [73]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl7/igt@kms_flip_scaled_crc@flip-32bpp-ytile-to-32bpp-ytileccs.html
    - shard-tglb:         NOTRUN -> [SKIP][74] ([i915#2587])
   [74]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-tglb7/igt@kms_flip_scaled_crc@flip-32bpp-ytile-to-32bpp-ytileccs.html
    - shard-apl:          NOTRUN -> [FAIL][75] ([i915#2641])
   [75]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-apl1/igt@kms_flip_scaled_crc@flip-32bpp-ytile-to-32bpp-ytileccs.html

  * igt@kms_frontbuffer_tracking@fbc-2p-scndscrn-cur-indfb-draw-blt:
    - shard-kbl:          NOTRUN -> [SKIP][76] ([fdo#109271]) +56 similar issues
   [76]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl1/igt@kms_frontbuffer_tracking@fbc-2p-scndscrn-cur-indfb-draw-blt.html

  * igt@kms_frontbuffer_tracking@fbcpsr-2p-primscrn-cur-indfb-draw-render:
    - shard-iclb:         NOTRUN -> [SKIP][77] ([fdo#109280]) +13 similar issues
   [77]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@kms_frontbuffer_tracking@fbcpsr-2p-primscrn-cur-indfb-draw-render.html

  * igt@kms_frontbuffer_tracking@fbcpsr-2p-scndscrn-pri-shrfb-draw-mmap-wc:
    - shard-tglb:         NOTRUN -> [SKIP][78] ([fdo#111825]) +2 similar issues
   [78]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-tglb7/igt@kms_frontbuffer_tracking@fbcpsr-2p-scndscrn-pri-shrfb-draw-mmap-wc.html

  * igt@kms_hdr@bpc-switch:
    - shard-skl:          [PASS][79] -> [FAIL][80] ([i915#1188])
   [79]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-skl1/igt@kms_hdr@bpc-switch.html
   [80]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-skl8/igt@kms_hdr@bpc-switch.html

  * igt@kms_invalid_dotclock:
    - shard-iclb:         NOTRUN -> [SKIP][81] ([fdo#109310])
   [81]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@kms_invalid_dotclock.html

  * igt@kms_pipe_crc_basic@suspend-read-crc-pipe-a:
    - shard-skl:          [PASS][82] -> [INCOMPLETE][83] ([i915#198]) +1 similar issue
   [82]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-skl9/igt@kms_pipe_crc_basic@suspend-read-crc-pipe-a.html
   [83]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-skl2/igt@kms_pipe_crc_basic@suspend-read-crc-pipe-a.html

  * igt@kms_plane@plane-panning-bottom-right-suspend-pipe-c-planes:
    - shard-iclb:         [PASS][84] -> [INCOMPLETE][85] ([i915#1185] / [i915#250])
   [84]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-iclb3/igt@kms_plane@plane-panning-bottom-right-suspend-pipe-c-planes.html
   [85]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb3/igt@kms_plane@plane-panning-bottom-right-suspend-pipe-c-planes.html

  * igt@kms_plane_alpha_blend@pipe-b-alpha-transparent-fb:
    - shard-kbl:          NOTRUN -> [FAIL][86] ([i915#265])
   [86]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl1/igt@kms_plane_alpha_blend@pipe-b-alpha-transparent-fb.html

  * igt@kms_plane_alpha_blend@pipe-d-constant-alpha-max:
    - shard-iclb:         NOTRUN -> [SKIP][87] ([fdo#109278]) +6 similar issues
   [87]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb8/igt@kms_plane_alpha_blend@pipe-d-constant-alpha-max.html

  * igt@kms_psr2_sf@plane-move-sf-dmg-area-3:
    - shard-kbl:          NOTRUN -> [SKIP][88] ([fdo#109271] / [i915#658])
   [88]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl1/igt@kms_psr2_sf@plane-move-sf-dmg-area-3.html

  * igt@kms_psr2_sf@primary-plane-update-sf-dmg-area-1:
    - shard-skl:          NOTRUN -> [SKIP][89] ([fdo#109271] / [i915#658])
   [89]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-skl3/igt@kms_psr2_sf@primary-plane-update-sf-dmg-area-1.html

  * igt@kms_psr2_sf@primary-plane-update-sf-dmg-area-5:
    - shard-iclb:         NOTRUN -> [SKIP][90] ([i915#658]) +1 similar issue
   [90]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@kms_psr2_sf@primary-plane-update-sf-dmg-area-5.html

  * igt@kms_psr2_su@page_flip:
    - shard-iclb:         [PASS][91] -> [SKIP][92] ([fdo#109642] / [fdo#111068] / [i915#658])
   [91]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-iclb2/igt@kms_psr2_su@page_flip.html
   [92]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@kms_psr2_su@page_flip.html

  * igt@kms_psr@psr2_cursor_mmap_gtt:
    - shard-hsw:          NOTRUN -> [SKIP][93] ([fdo#109271] / [i915#1072]) +1 similar issue
   [93]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-hsw1/igt@kms_psr@psr2_cursor_mmap_gtt.html

  * igt@kms_psr@psr2_primary_mmap_cpu:
    - shard-iclb:         [PASS][94] -> [SKIP][95] ([fdo#109441]) +1 similar issue
   [94]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-iclb2/igt@kms_psr@psr2_primary_mmap_cpu.html
   [95]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb4/igt@kms_psr@psr2_primary_mmap_cpu.html

  * igt@kms_psr@psr2_primary_render:
    - shard-iclb:         NOTRUN -> [SKIP][96] ([fdo#109441])
   [96]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb8/igt@kms_psr@psr2_primary_render.html

  * igt@kms_vblank@pipe-a-ts-continuation-suspend:
    - shard-kbl:          [PASS][97] -> [DMESG-WARN][98] ([i915#180] / [i915#295])
   [97]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-kbl2/igt@kms_vblank@pipe-a-ts-continuation-suspend.html
   [98]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl7/igt@kms_vblank@pipe-a-ts-continuation-suspend.html

  * igt@kms_writeback@writeback-pixel-formats:
    - shard-iclb:         NOTRUN -> [SKIP][99] ([i915#2437])
   [99]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb8/igt@kms_writeback@writeback-pixel-formats.html

  * igt@nouveau_crc@pipe-b-ctx-flip-skip-current-frame:
    - shard-apl:          NOTRUN -> [SKIP][100] ([fdo#109271]) +41 similar issues
   [100]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-apl7/igt@nouveau_crc@pipe-b-ctx-flip-skip-current-frame.html

  * igt@nouveau_crc@pipe-b-source-outp-complete:
    - shard-iclb:         NOTRUN -> [SKIP][101] ([i915#2530]) +1 similar issue
   [101]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb8/igt@nouveau_crc@pipe-b-source-outp-complete.html

  * igt@nouveau_crc@pipe-d-ctx-flip-detection:
    - shard-skl:          NOTRUN -> [SKIP][102] ([fdo#109271]) +27 similar issues
   [102]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-skl3/igt@nouveau_crc@pipe-d-ctx-flip-detection.html

  * igt@prime_nv_api@i915_self_import_to_different_fd:
    - shard-tglb:         NOTRUN -> [SKIP][103] ([fdo#109291])
   [103]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-tglb7/igt@prime_nv_api@i915_self_import_to_different_fd.html

  * igt@prime_nv_api@nv_self_import_to_different_fd:
    - shard-iclb:         NOTRUN -> [SKIP][104] ([fdo#109291]) +1 similar issue
   [104]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@prime_nv_api@nv_self_import_to_different_fd.html

  * igt@sysfs_clients@split-25@vcs0:
    - shard-skl:          [PASS][105] -> [SKIP][106] ([fdo#109271])
   [105]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-skl9/igt@sysfs_clients@split-25@vcs0.html
   [106]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-skl1/igt@sysfs_clients@split-25@vcs0.html

  * igt@sysfs_heartbeat_interval@nopreempt@bcs0:
    - shard-apl:          [PASS][107] -> [FAIL][108] ([i915#1778]) +1 similar issue
   [107]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-apl2/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html
   [108]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-apl7/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html
    - shard-iclb:         [PASS][109] -> [FAIL][110] ([i915#1778]) +1 similar issue
   [109]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-iclb3/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html
   [110]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb2/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html
    - shard-glk:          [PASS][111] -> [FAIL][112] ([i915#1778]) +1 similar issue
   [111]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-glk2/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html
   [112]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-glk6/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html
    - shard-kbl:          [PASS][113] -> [FAIL][114] ([i915#1778]) +1 similar issue
   [113]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-kbl3/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html
   [114]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl1/igt@sysfs_heartbeat_interval@nopreempt@bcs0.html

  * igt@sysfs_heartbeat_interval@nopreempt@rcs0:
    - shard-skl:          [PASS][115] -> [FAIL][116] ([i915#1778]) +1 similar issue
   [115]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-skl4/igt@sysfs_heartbeat_interval@nopreempt@rcs0.html
   [116]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-skl5/igt@sysfs_heartbeat_interval@nopreempt@rcs0.html

  * igt@sysfs_heartbeat_interval@nopreempt@vcs0:
    - shard-hsw:          NOTRUN -> [SKIP][117] ([fdo#109271]) +41 similar issues
   [117]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-hsw1/igt@sysfs_heartbeat_interval@nopreempt@vcs0.html

  * igt@sysfs_heartbeat_interval@nopreempt@vcs1:
    - shard-tglb:         [PASS][118] -> [FAIL][119] ([i915#1778]) +4 similar issues
   [118]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-tglb2/igt@sysfs_heartbeat_interval@nopreempt@vcs1.html
   [119]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-tglb5/igt@sysfs_heartbeat_interval@nopreempt@vcs1.html

  
#### Possible fixes ####

  * igt@gem_ctx_persistence@close-replace-race:
    - shard-glk:          [TIMEOUT][120] ([i915#2918]) -> [PASS][121]
   [120]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-glk5/igt@gem_ctx_persistence@close-replace-race.html
   [121]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-glk2/igt@gem_ctx_persistence@close-replace-race.html

  * igt@gem_exec_balancer@hang:
    - shard-iclb:         [INCOMPLETE][122] ([i915#1895] / [i915#2295] / [i915#3031]) -> [PASS][123]
   [122]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-iclb1/igt@gem_exec_balancer@hang.html
   [123]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb8/igt@gem_exec_balancer@hang.html

  * igt@gem_exec_fair@basic-deadline:
    - shard-kbl:          [FAIL][124] ([i915#2846]) -> [PASS][125]
   [124]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-kbl1/igt@gem_exec_fair@basic-deadline.html
   [125]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl2/igt@gem_exec_fair@basic-deadline.html

  * igt@gem_exec_fair@basic-none-solo@rcs0:
    - shard-kbl:          [FAIL][126] ([i915#2842]) -> [PASS][127] +1 similar issue
   [126]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-kbl6/igt@gem_exec_fair@basic-none-solo@rcs0.html
   [127]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl2/igt@gem_exec_fair@basic-none-solo@rcs0.html

  * igt@gem_exec_fair@basic-pace-share@rcs0:
    - shard-tglb:         [FAIL][128] ([i915#2842]) -> [PASS][129]
   [128]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-tglb1/igt@gem_exec_fair@basic-pace-share@rcs0.html
   [129]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-tglb3/igt@gem_exec_fair@basic-pace-share@rcs0.html

  * igt@gem_exec_fair@basic-throttle@rcs0:
    - shard-iclb:         [FAIL][130] ([i915#2849]) -> [PASS][131]
   [130]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-iclb2/igt@gem_exec_fair@basic-throttle@rcs0.html
   [131]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-iclb5/igt@gem_exec_fair@basic-throttle@rcs0.html

  * igt@gem_exec_schedule@u-fairslice@bcs0:
    - shard-tglb:         [DMESG-WARN][132] ([i915#2803]) -> [PASS][133]
   [132]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-tglb6/igt@gem_exec_schedule@u-fairslice@bcs0.html
   [133]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-tglb7/igt@gem_exec_schedule@u-fairslice@bcs0.html

  * igt@gem_exec_schedule@u-fairslice@rcs0:
    - shard-apl:          [DMESG-WARN][134] ([i915#1610]) -> [PASS][135]
   [134]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-apl7/igt@gem_exec_schedule@u-fairslice@rcs0.html
   [135]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-apl1/igt@gem_exec_schedule@u-fairslice@rcs0.html

  * igt@gem_exec_schedule@u-fairslice@vecs0:
    - shard-kbl:          [DMESG-WARN][136] ([i915#1610] / [i915#2803]) -> [PASS][137]
   [136]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-kbl4/igt@gem_exec_schedule@u-fairslice@vecs0.html
   [137]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-kbl7/igt@gem_exec_schedule@u-fairslice@vecs0.html

  * igt@i915_pm_rc6_residency@rc6-idle:
    - shard-hsw:          [WARN][138] ([i915#1519]) -> [PASS][139]
   [138]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-hsw6/igt@i915_pm_rc6_residency@rc6-idle.html
   [139]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-hsw5/igt@i915_pm_rc6_residency@rc6-idle.html

  * igt@i915_selftest@live@hangcheck:
    - shard-skl:          [INCOMPLETE][140] ([i915#2782]) -> [PASS][141]
   [140]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_9769/shard-skl4/igt@i915_selftest@live@hangcheck.html
   [141]: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/shard-skl10/igt@i915_selftest@live@hangcheck.html

  * igt@kms_cursor_crc@pipe-b-cursor-suspend:
    - shard-kbl:          [DMESG-WARN][142] ([i915#180]) -> [PASS][143] +1 similar

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_19669/index.html

[-- Attachment #1.2: Type: text/html, Size: 33525 bytes --]

[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply	[flat|nested] 21+ messages in thread

* [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing
@ 2021-02-05  1:17 Chris Wilson
  0 siblings, 0 replies; 21+ messages in thread
From: Chris Wilson @ 2021-02-05  1:17 UTC (permalink / raw)
  To: intel-gfx; +Cc: Chris Wilson

The heartbeat runs through a few phases that we expect to complete
within a certain number of heartbeat intervals. First we must submit the
heartbeat to the queue, and if the queue is occupied it may take a
couple of intervals before the heartbeat preempts the workload and is
submitted to HW. Once running on HW, completion is not instantaneous as
it may have to first reset the current workload before it itself runs
through the empty request and signals completion. As such, we know that
the heartbeat must take at least the preempt reset timeout and before we
have had a chance to reset the engine, we do not want to issue a global
reset ourselves (simply so that we only try to do one reset at a time
and not confuse ourselves by resetting twice and hitting an innocent.)

So by taking into consideration that once running the request must take
a finite amount of time, we can delay the final completion check to
accommodate that and avoid checking too early (before we've had a chance
to handle any engine resets required).

v2: Attach a callback to flush the work immediately upon the heartbeat
completion and insert the delay before the next.

Suggested-by: CQ Tang <cq.tang@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
 .../gpu/drm/i915/gt/intel_engine_heartbeat.c  | 85 +++++++++++++++++--
 drivers/gpu/drm/i915/gt/intel_engine_types.h  |  1 +
 .../drm/i915/gt/selftest_engine_heartbeat.c   | 30 +++++--
 3 files changed, 99 insertions(+), 17 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
index 93741a65924a..3df10bfafb3b 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
@@ -20,6 +20,18 @@
  * issue a reset -- in the hope that restores progress.
  */
 
+#define HEARTBEAT_COMPLETION 50u /* milliseconds */
+
+static long completion_timeout(const struct intel_engine_cs *engine)
+{
+	long timeout = HEARTBEAT_COMPLETION;
+
+	if (intel_engine_has_preempt_reset(engine))
+		timeout += READ_ONCE(engine->props.preempt_timeout_ms);
+
+	return msecs_to_jiffies(timeout);
+}
+
 static bool next_heartbeat(struct intel_engine_cs *engine)
 {
 	long delay;
@@ -29,6 +41,28 @@ static bool next_heartbeat(struct intel_engine_cs *engine)
 		return false;
 
 	delay = msecs_to_jiffies_timeout(delay);
+
+	/*
+	 * Once we submit a heartbeat to the HW, we know that it will take
+	 * at least a certain amount of time to complete. On a hanging system
+	 * it will first have to wait for the preempt reset timeout, and
+	 * then it will take some time for the reset to resume with the
+	 * heartbeat and for it to complete. So once we have submitted the
+	 * heartbeat to HW, we can wait a while longer before declaring the
+	 * engine stuck and forcing a reset ourselves. If we do a reset
+	 * and the engine is also doing a reset, it is possible that we
+	 * reset the engine twice, harming an innocent.
+	 *
+	 * Before we have sumitted the heartbeat, we do not want to change
+	 * the interval as we to promote the heartbeat and trigger preemption
+	 * in a deterministic time frame.
+	 */
+	if (engine->heartbeat.systole) {
+		intel_engine_flush_submission(engine);
+		if (i915_request_is_active(engine->heartbeat.systole))
+			delay = max(delay, completion_timeout(engine));
+	}
+
 	if (delay >= HZ)
 		delay = round_jiffies_up_relative(delay);
 	mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, delay + 1);
@@ -48,12 +82,44 @@ heartbeat_create(struct intel_context *ce, gfp_t gfp)
 	return rq;
 }
 
+static void defibrillator(struct dma_fence *f, struct dma_fence_cb *cb)
+{
+	struct intel_engine_cs *engine =
+		container_of(cb, typeof(*engine), heartbeat.cb);
+
+	if (READ_ONCE(engine->heartbeat.systole))
+		mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, 0);
+}
+
+static void
+track_heartbeat(struct intel_engine_cs *engine, struct i915_request *rq)
+{
+	engine->heartbeat.systole = i915_request_get(rq);
+	if (dma_fence_add_callback(&rq->fence,
+				   &engine->heartbeat.cb,
+				   defibrillator))
+		mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, 0);
+}
+
+static void
+untrack_heartbeat(struct intel_engine_cs *engine)
+{
+	struct i915_request *rq;
+
+	rq = fetch_and_zero(&engine->heartbeat.systole);
+	if (!rq)
+		return;
+
+	dma_fence_remove_callback(&rq->fence, &engine->heartbeat.cb);
+	i915_request_put(rq);
+}
+
 static void idle_pulse(struct intel_engine_cs *engine, struct i915_request *rq)
 {
 	engine->wakeref_serial = READ_ONCE(engine->serial) + 1;
 	i915_request_add_active_barriers(rq);
 	if (!engine->heartbeat.systole && intel_engine_has_heartbeat(engine))
-		engine->heartbeat.systole = i915_request_get(rq);
+		track_heartbeat(engine, rq);
 }
 
 static void heartbeat_commit(struct i915_request *rq,
@@ -91,10 +157,8 @@ static void heartbeat(struct work_struct *wrk)
 	intel_engine_flush_submission(engine);
 
 	rq = engine->heartbeat.systole;
-	if (rq && i915_request_completed(rq)) {
-		i915_request_put(rq);
-		engine->heartbeat.systole = NULL;
-	}
+	if (rq && i915_request_completed(rq))
+		untrack_heartbeat(engine);
 
 	if (!intel_engine_pm_get_if_awake(engine))
 		return;
@@ -151,6 +215,11 @@ static void heartbeat(struct work_struct *wrk)
 		goto out;
 	}
 
+	/* Just completed one heartbeat, wait a tick before the next */
+	if (rq)
+		goto out;
+
+	/* The engine is parking. We can rest until the next user */
 	serial = READ_ONCE(engine->serial);
 	if (engine->wakeref_serial == serial)
 		goto out;
@@ -175,7 +244,7 @@ static void heartbeat(struct work_struct *wrk)
 	mutex_unlock(&ce->timeline->mutex);
 out:
 	if (!engine->i915->params.enable_hangcheck || !next_heartbeat(engine))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+		untrack_heartbeat(engine);
 	intel_engine_pm_put(engine);
 }
 
@@ -189,8 +258,8 @@ void intel_engine_unpark_heartbeat(struct intel_engine_cs *engine)
 
 void intel_engine_park_heartbeat(struct intel_engine_cs *engine)
 {
-	if (cancel_delayed_work(&engine->heartbeat.work))
-		i915_request_put(fetch_and_zero(&engine->heartbeat.systole));
+	while (cancel_delayed_work(&engine->heartbeat.work))
+		untrack_heartbeat(engine); /* completion may rearm work */
 }
 
 void intel_engine_init_heartbeat(struct intel_engine_cs *engine)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 7159f9575e65..4956594c8b93 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -347,6 +347,7 @@ struct intel_engine_cs {
 	struct {
 		struct delayed_work work;
 		struct i915_request *systole;
+		struct dma_fence_cb cb;
 		unsigned long blocked;
 	} heartbeat;
 
diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
index b2c369317bf1..fbc1ef8aeeb4 100644
--- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c
@@ -202,7 +202,8 @@ static int cmp_u32(const void *_a, const void *_b)
 
 static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 {
-	const unsigned int error_threshold = max(20000u, jiffies_to_usecs(6));
+	const unsigned int error_threshold =
+		max(3 * HEARTBEAT_COMPLETION * 1000, jiffies_to_usecs(6));
 	struct intel_context *ce;
 	struct i915_request *rq;
 	ktime_t t0, t1;
@@ -221,27 +222,37 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 		goto err_pm;
 
 	for (i = 0; i < ARRAY_SIZE(times); i++) {
+		unsigned long timeout;
+
+		timeout = jiffies + 2;
 		do {
+			unsigned long serial;
+
 			/* Manufacture a tick */
 			intel_engine_park_heartbeat(engine);
 			GEM_BUG_ON(engine->heartbeat.systole);
-			engine->serial++; /*  pretend we are not idle! */
+			serial = ++engine->serial; /* pretend we are not idle */
 			intel_engine_unpark_heartbeat(engine);
 
 			flush_delayed_work(&engine->heartbeat.work);
-			if (!delayed_work_pending(&engine->heartbeat.work)) {
-				pr_err("%s: heartbeat %d did not start\n",
-				       engine->name, i);
-				err = -EINVAL;
-				goto err_pm;
-			}
 
 			rcu_read_lock();
 			rq = READ_ONCE(engine->heartbeat.systole);
 			if (rq)
 				rq = i915_request_get_rcu(rq);
 			rcu_read_unlock();
-		} while (!rq);
+			if (rq)
+				break;
+
+			if (engine->serial == serial) {
+				pr_err("%s: heartbeat %d did not start\n",
+				       engine->name, i);
+				err = -EINVAL;
+				goto err_reset;
+			}
+		} while (!rq && !time_after(jiffies, timeout));
+		if (!rq) /* Too fast! */
+			goto err_reset;
 
 		t0 = ktime_get();
 		while (rq == READ_ONCE(engine->heartbeat.systole))
@@ -275,6 +286,7 @@ static int __live_heartbeat_fast(struct intel_engine_cs *engine)
 		err = -EINVAL;
 	}
 
+err_reset:
 	reset_heartbeat(engine);
 err_pm:
 	intel_engine_pm_put(engine);
-- 
2.20.1

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply related	[flat|nested] 21+ messages in thread

* [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing
@ 2021-02-03 21:06 Chris Wilson
  0 siblings, 0 replies; 21+ messages in thread
From: Chris Wilson @ 2021-02-03 21:06 UTC (permalink / raw)
  To: intel-gfx; +Cc: Chris Wilson

The heartbeat runs through a few phases that we expect to complete
within a certain number of heartbeat intervals. First we must submit the
heartbeat to the queue, and if the queue is occupied it may take a
couple of intervals before the heartbeat preempts the workload and is
submitted to HW. Once running on HW, completion is not instantaneous as
it may have to first reset the current workload before it itself runs
through the empty request and signals completion. As such, we know that
the heartbeat must take at least the preempt reset timeout and before we
have had a chance to reset the engine, we do not want to issue a global
reset ourselves (simply so that we only try to do one reset at a time
and not confuse ourselves by resetting twice and hitting an innocent.)

So by taking into consideration that once running the request must take
a finite amount of time, we can delay the final completion check to
accomodate that and avoid checking too early (before we've had a chance
to handle any engine resets required).

Suggested-by: CQ Tang <cq.tang@intel.com>
Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>
---
 .../gpu/drm/i915/gt/intel_engine_heartbeat.c  | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
index 778bcae5ef2c..2a999911f4ac 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c
@@ -20,6 +20,16 @@
  * issue a reset -- in the hope that restores progress.
  */
 
+static long completion_timeout(const struct intel_engine_cs *engine)
+{
+	long timeout = 50;
+
+	if (intel_engine_has_preempt_reset(engine))
+		timeout += READ_ONCE(engine->props.heartbeat_interval_ms);
+
+	return msecs_to_jiffies(timeout);
+}
+
 static bool next_heartbeat(struct intel_engine_cs *engine)
 {
 	long delay;
@@ -29,6 +39,28 @@ static bool next_heartbeat(struct intel_engine_cs *engine)
 		return false;
 
 	delay = msecs_to_jiffies_timeout(delay);
+
+	/*
+	 * Once we submit a heartbeat to the HW, we know that it will take
+	 * at least a certain amount of time to complete. On a hanging system
+	 * it will first have to wait for the preempt reset timeout, and
+	 * then it will take some time for the reset to resume with the
+	 * heartbeat and for it to complete. So once we have submitted the
+	 * heartbeat to HW, we can wait a while longer before declaring the
+	 * engine stuck and forcing a reset ourselves. If we do a reset
+	 * and the engine is also doing a reset, it is possible that we
+	 * reset the engine twice, harming an innocent.
+	 *
+	 * Before we have sumitted the heartbeat, we do not want to change
+	 * the interval as we to promote the heartbeat and trigger preemption
+	 * in a deterministic time frame.
+	 */
+	if (engine->heartbeat.systole) {
+		intel_engine_flush_submission(engine);
+		if (i915_request_is_active(engine->heartbeat.systole))
+			delay = max(delay, completion_timeout(engine));
+	}
+
 	if (delay >= HZ)
 		delay = round_jiffies_up_relative(delay);
 	mod_delayed_work(system_highpri_wq, &engine->heartbeat.work, delay);
-- 
2.20.1

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply related	[flat|nested] 21+ messages in thread

end of thread, other threads:[~2021-02-12 15:08 UTC | newest]

Thread overview: 21+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-02-05 10:49 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
2021-02-05 11:15 ` [Intel-gfx] [PATCH v4] " Chris Wilson
2021-02-05 13:15 ` [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev5) Patchwork
2021-02-05 14:22 ` [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
2021-02-05 14:32 ` Chris Wilson
2021-02-10 22:38   ` Tang, CQ
2021-02-05 20:23 ` [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev7) Patchwork
2021-02-06  6:45 ` [Intel-gfx] ✓ Fi.CI.IGT: " Patchwork
2021-02-11 12:24 ` [Intel-gfx] [PATCH v3] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
2021-02-11 14:59 ` [Intel-gfx] ✗ Fi.CI.BAT: failure for drm/i915/gt: Ratelimit heartbeat completion probing (rev8) Patchwork
2021-02-11 21:06 ` [Intel-gfx] [PATCH v4] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
2021-02-11 22:02 ` [Intel-gfx] ✗ Fi.CI.BAT: failure for drm/i915/gt: Ratelimit heartbeat completion probing (rev9) Patchwork
2021-02-11 23:14 ` [Intel-gfx] [PATCH v5] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
2021-02-11 23:54   ` Tang, CQ
2021-02-12  2:58 ` [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev10) Patchwork
2021-02-12  5:26 ` [Intel-gfx] ✗ Fi.CI.IGT: failure " Patchwork
2021-02-12 12:58 ` [Intel-gfx] [PATCH v6] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
2021-02-12 14:00 ` [Intel-gfx] ✓ Fi.CI.BAT: success for drm/i915/gt: Ratelimit heartbeat completion probing (rev11) Patchwork
2021-02-12 15:08 ` [Intel-gfx] ✗ Fi.CI.IGT: failure " Patchwork
  -- strict thread matches above, loose matches on Subject: below --
2021-02-05  1:17 [Intel-gfx] [PATCH] drm/i915/gt: Ratelimit heartbeat completion probing Chris Wilson
2021-02-03 21:06 Chris Wilson

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.