[Intel-gfx] [PATCH v2 1/2] drm/i915: Reboot CI if we get wedged during driver init

* [Intel-gfx] [PATCH v2 1/2] drm/i915: Reboot CI if we get wedged during driver init
@ 2020-07-06 10:49 Michał Winiarski
  2020-07-06 10:49 ` [Intel-gfx] [PATCH 2/2] drm/i915: Print caller when tainting for CI Michał Winiarski
                   ` (4 more replies)
  0 siblings, 5 replies; 7+ messages in thread
From: Michał Winiarski @ 2020-07-06 10:49 UTC (permalink / raw)
  To: intel-gfx; +Cc: Michał Winiarski, Chris Wilson

From: Michał Winiarski <michal.winiarski@intel.com>

Getting wedged device on driver init is pretty much unrecoverable.
Since we're running various scenarios that may potentially hit this in
CI (module reload / selftests / hotunplug), and if it happens, it means
that we can't trust any subsequent CI results, we should just apply the
taint to let the CI know that it should reboot (CI checks taint between
test runs).

v2: Comment that WEDGED_ON_INIT is non-recoverable, distinguish
    WEDGED_ON_INIT from WEDGED_ON_FINI (Chris)

Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Michal Wajdeczko <michal.wajdeczko@intel.com>
Cc: Petri Latvala <petri.latvala@intel.com>
---
 drivers/gpu/drm/i915/gt/intel_engine_user.c |  2 +-
 drivers/gpu/drm/i915/gt/intel_gt.c          |  2 +-
 drivers/gpu/drm/i915/gt/intel_gt.h          | 12 ++++++++----
 drivers/gpu/drm/i915/gt/intel_gt_pm.c       |  2 +-
 drivers/gpu/drm/i915/gt/intel_reset.c       | 13 +++++++++++--
 drivers/gpu/drm/i915/gt/intel_reset.h       | 10 ++--------
 drivers/gpu/drm/i915/gt/intel_reset_types.h |  7 ++++++-
 7 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/drivers/gpu/drm/i915/gt/intel_engine_user.c b/drivers/gpu/drm/i915/gt/intel_engine_user.c
index 848decee9066..34e6096f196e 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_user.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_user.c
@@ -201,7 +201,7 @@ void intel_engines_driver_register(struct drm_i915_private *i915)
 				     uabi_node);
 		char old[sizeof(engine->name)];
 
-		if (intel_gt_has_init_error(engine->gt))
+		if (intel_gt_has_unrecoverable_error(engine->gt))
 			continue; /* ignore incomplete engines */
 
 		GEM_BUG_ON(engine->class >= ARRAY_SIZE(uabi_classes));
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
index ebc29b6ee86c..876f78759095 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt.c
@@ -510,7 +510,7 @@ static int __engines_verify_workarounds(struct intel_gt *gt)
 
 static void __intel_gt_disable(struct intel_gt *gt)
 {
-	intel_gt_set_wedged_on_init(gt);
+	intel_gt_set_wedged_on_fini(gt);
 
 	intel_gt_suspend_prepare(gt);
 	intel_gt_suspend_late(gt);
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.h b/drivers/gpu/drm/i915/gt/intel_gt.h
index 4fac043750aa..7201f96723d8 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt.h
@@ -58,14 +58,18 @@ static inline u32 intel_gt_scratch_offset(const struct intel_gt *gt,
 	return i915_ggtt_offset(gt->scratch) + field;
 }
 
-static inline bool intel_gt_is_wedged(const struct intel_gt *gt)
+static inline bool intel_gt_has_unrecoverable_error(const struct intel_gt *gt)
 {
-	return __intel_reset_failed(&gt->reset);
+	return test_bit(I915_WEDGED_ON_INIT, &gt->reset.flags) ||
+               test_bit(I915_WEDGED_ON_FINI, &gt->reset.flags);
 }
 
-static inline bool intel_gt_has_init_error(const struct intel_gt *gt)
+static inline bool intel_gt_is_wedged(const struct intel_gt *gt)
 {
-	return test_bit(I915_WEDGED_ON_INIT, &gt->reset.flags);
+	GEM_BUG_ON(intel_gt_has_unrecoverable_error(gt) ?
+		   !test_bit(I915_WEDGED, &gt->reset.flags) : false);
+
+	return unlikely(test_bit(I915_WEDGED, &gt->reset.flags));
 }
 
 #endif /* __INTEL_GT_H__ */
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
index f1d5333f9456..274aa0dd7050 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c
@@ -188,7 +188,7 @@ int intel_gt_resume(struct intel_gt *gt)
 	enum intel_engine_id id;
 	int err;
 
-	err = intel_gt_has_init_error(gt);
+	err = intel_gt_has_unrecoverable_error(gt);
 	if (err)
 		return err;
 
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c
index 0156f1f5c736..6f94b6479a2f 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.c
+++ b/drivers/gpu/drm/i915/gt/intel_reset.c
@@ -880,7 +880,7 @@ static bool __intel_gt_unset_wedged(struct intel_gt *gt)
 		return true;
 
 	/* Never fully initialised, recovery impossible */
-	if (test_bit(I915_WEDGED_ON_INIT, &gt->reset.flags))
+	if (intel_gt_has_unrecoverable_error(gt))
 		return false;
 
 	GT_TRACE(gt, "start\n");
@@ -1342,7 +1342,7 @@ int intel_gt_terminally_wedged(struct intel_gt *gt)
 	if (!intel_gt_is_wedged(gt))
 		return 0;
 
-	if (intel_gt_has_init_error(gt))
+	if (intel_gt_has_unrecoverable_error(gt))
 		return -EIO;
 
 	/* Reset still in progress? Maybe we will recover? */
@@ -1360,6 +1360,15 @@ void intel_gt_set_wedged_on_init(struct intel_gt *gt)
 		     I915_WEDGED_ON_INIT);
 	intel_gt_set_wedged(gt);
 	set_bit(I915_WEDGED_ON_INIT, &gt->reset.flags);
+
+	/* Wedged on init is non-recoverable */
+	add_taint_for_CI(TAINT_WARN);
+}
+
+void intel_gt_set_wedged_on_fini(struct intel_gt *gt)
+{
+	intel_gt_set_wedged(gt);
+	set_bit(I915_WEDGED_ON_FINI, &gt->reset.flags);
 }
 
 void intel_gt_init_reset(struct intel_gt *gt)
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.h b/drivers/gpu/drm/i915/gt/intel_reset.h
index 8e8d5f761166..a0eec7c11c0c 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.h
+++ b/drivers/gpu/drm/i915/gt/intel_reset.h
@@ -47,8 +47,10 @@ int intel_gt_terminally_wedged(struct intel_gt *gt);
 /*
  * There's no unset_wedged_on_init paired with this one.
  * Once we're wedged on init, there's no going back.
+ * Same thing for unset_wedged_on_fini.
  */
 void intel_gt_set_wedged_on_init(struct intel_gt *gt);
+void intel_gt_set_wedged_on_fini(struct intel_gt *gt);
 
 int __intel_gt_reset(struct intel_gt *gt, intel_engine_mask_t engine_mask);
 
@@ -71,14 +73,6 @@ void __intel_fini_wedge(struct intel_wedge_me *w);
 	     (W)->gt;							\
 	     __intel_fini_wedge((W)))
 
-static inline bool __intel_reset_failed(const struct intel_reset *reset)
-{
-	GEM_BUG_ON(test_bit(I915_WEDGED_ON_INIT, &reset->flags) ?
-		   !test_bit(I915_WEDGED, &reset->flags) : false);
-
-	return unlikely(test_bit(I915_WEDGED, &reset->flags));
-}
-
 bool intel_has_gpu_reset(const struct intel_gt *gt);
 bool intel_has_reset_engine(const struct intel_gt *gt);
 
diff --git a/drivers/gpu/drm/i915/gt/intel_reset_types.h b/drivers/gpu/drm/i915/gt/intel_reset_types.h
index f43bc3a0fe4f..fe386bf5306c 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_reset_types.h
@@ -34,12 +34,17 @@ struct intel_reset {
 	 * longer use the GPU - similar to #I915_WEDGED bit. The difference in
 	 * in the way we're handling "forced" unwedged (e.g. through debugfs),
 	 * which is not allowed in case we failed to initialize.
+         *
+         * #I915_WEDGED_ON_FINI - Similar to #I915_WEDGED_ON_INIT, except we
+         * use it to mark that the GPU is no longer available (and prevent
+         * users from using it).
 	 */
 	unsigned long flags;
 #define I915_RESET_BACKOFF	0
 #define I915_RESET_MODESET	1
 #define I915_RESET_ENGINE	2
-#define I915_WEDGED_ON_INIT	(BITS_PER_LONG - 2)
+#define I915_WEDGED_ON_INIT	(BITS_PER_LONG - 3)
+#define I915_WEDGED_ON_FINI	(BITS_PER_LONG - 2)
 #define I915_WEDGED		(BITS_PER_LONG - 1)
 
 	struct mutex mutex; /* serialises wedging/unwedging */
-- 
2.27.0

_______________________________________________
Intel-gfx mailing list
Intel-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

^ permalink raw reply related	[flat|nested] 7+ messages in thread