From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S932904AbdDFXYE (ORCPT ); Thu, 6 Apr 2017 19:24:04 -0400 Received: from mx1.redhat.com ([209.132.183.28]:34069 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1755320AbdDFXXu (ORCPT ); Thu, 6 Apr 2017 19:23:50 -0400 DMARC-Filter: OpenDMARC Filter v1.3.2 mx1.redhat.com EE2B166CAB Authentication-Results: ext-mx09.extmail.prod.ext.phx2.redhat.com; dmarc=none (p=none dis=none) header.from=redhat.com Authentication-Results: ext-mx09.extmail.prod.ext.phx2.redhat.com; spf=pass smtp.mailfrom=aarcange@redhat.com DKIM-Filter: OpenDKIM Filter v2.11.0 mx1.redhat.com EE2B166CAB From: Andrea Arcangeli To: Martin Kepplinger , Thorsten Leemhuis , daniel.vetter@intel.com, Dave Airlie , Chris Wilson Cc: intel-gfx@lists.freedesktop.org, linux-kernel@vger.kernel.org, dri-devel@lists.freedesktop.org Subject: [PATCH 1/5] i915: avoid kernel hang caused by synchronize rcu struct_mutex deadlock Date: Fri, 7 Apr 2017 01:23:43 +0200 Message-Id: <20170406232347.988-2-aarcange@redhat.com> In-Reply-To: <20170406232347.988-1-aarcange@redhat.com> References: <87pogtplxr.fsf@intel.com> <20170406232347.988-1-aarcange@redhat.com> X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.5.110.38]); Thu, 06 Apr 2017 23:23:50 +0000 (UTC) Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org synchronize_rcu/synchronize_sched/synchronize_rcu_expedited() will hang until its own workqueues are run. The i915 gem workqueues will wait on the struct_mutex to be released. So we cannot wait for a quiescent state using those rcu primitives while holding the struct_mutex or it creates a circular lock dependency resulting in kernel hangs (which is reproducible but goes undetected by lockdep). This started in commit 3d3d18f086cdda72ee18a454db70ca72c6e3246c and lockdep didn't detect it apparently. kswapd0 D 0 700 2 0x00000000 Call Trace: ? __schedule+0x1a5/0x660 ? schedule+0x36/0x80 ? _synchronize_rcu_expedited.constprop.65+0x2ef/0x300 ? wake_up_bit+0x20/0x20 ? rcu_stall_kick_kthreads.part.54+0xc0/0xc0 ? rcu_exp_wait_wake+0x530/0x530 ? i915_gem_shrink+0x34b/0x4b0 ? i915_gem_shrinker_scan+0x7c/0x90 ? i915_gem_shrinker_scan+0x7c/0x90 ? shrink_slab.part.61.constprop.72+0x1c1/0x3a0 ? shrink_zone+0x154/0x160 ? kswapd+0x40a/0x720 ? kthread+0xf4/0x130 ? try_to_free_pages+0x450/0x450 ? kthread_create_on_node+0x40/0x40 ? ret_from_fork+0x23/0x30 plasmashell D 0 4657 4614 0x00000000 Call Trace: ? __schedule+0x1a5/0x660 ? schedule+0x36/0x80 ? schedule_preempt_disabled+0xe/0x10 ? __mutex_lock.isra.4+0x1c9/0x790 ? i915_gem_close_object+0x26/0xc0 ? i915_gem_close_object+0x26/0xc0 ? drm_gem_object_release_handle+0x48/0x90 ? drm_gem_handle_delete+0x50/0x80 ? drm_ioctl+0x1fa/0x420 ? drm_gem_handle_create+0x40/0x40 ? pipe_write+0x391/0x410 ? __vfs_write+0xc6/0x120 ? do_vfs_ioctl+0x8b/0x5d0 ? SyS_ioctl+0x3b/0x70 ? entry_SYSCALL_64_fastpath+0x13/0x94 kworker/0:0 D 0 29186 2 0x00000000 Workqueue: events __i915_gem_free_work Call Trace: ? __schedule+0x1a5/0x660 ? schedule+0x36/0x80 ? schedule_preempt_disabled+0xe/0x10 ? __mutex_lock.isra.4+0x1c9/0x790 ? del_timer_sync+0x44/0x50 ? update_curr+0x57/0x110 ? __i915_gem_free_objects+0x31/0x300 ? __i915_gem_free_objects+0x31/0x300 ? __i915_gem_free_work+0x2d/0x40 ? process_one_work+0x13a/0x3b0 ? worker_thread+0x4a/0x460 ? kthread+0xf4/0x130 ? process_one_work+0x3b0/0x3b0 ? kthread_create_on_node+0x40/0x40 ? ret_from_fork+0x23/0x30 Signed-off-by: Andrea Arcangeli --- drivers/gpu/drm/i915/i915_gem.c | 9 +++++++++ drivers/gpu/drm/i915/i915_gem_shrinker.c | 14 ++++++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 67b1fc5..3982489 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -4742,6 +4742,13 @@ int i915_gem_freeze(struct drm_i915_private *dev_priv) i915_gem_shrink_all(dev_priv); mutex_unlock(&dev_priv->drm.struct_mutex); + /* + * Cannot call synchronize_rcu() inside the struct_mutex + * because it may block until workqueues complete, and the + * running workqueue may wait on the struct_mutex. + */ + synchronize_rcu(); /* wait for our earlier RCU delayed slab frees */ + intel_runtime_pm_put(dev_priv); return 0; @@ -4781,6 +4788,8 @@ int i915_gem_freeze_late(struct drm_i915_private *dev_priv) } mutex_unlock(&dev_priv->drm.struct_mutex); + synchronize_rcu_expedited(); + return 0; } diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c index d5d2b4c..fea1454 100644 --- a/drivers/gpu/drm/i915/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c @@ -235,9 +235,6 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, if (unlock) mutex_unlock(&dev_priv->drm.struct_mutex); - /* expedite the RCU grace period to free some request slabs */ - synchronize_rcu_expedited(); - return count; } @@ -263,7 +260,6 @@ unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv) I915_SHRINK_BOUND | I915_SHRINK_UNBOUND | I915_SHRINK_ACTIVE); - synchronize_rcu(); /* wait for our earlier RCU delayed slab frees */ return freed; } @@ -324,6 +320,16 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) if (unlock) mutex_unlock(&dev->struct_mutex); + if (likely(__mutex_owner(&dev->struct_mutex) != current)) + /* + * If reclaim was invoked by an allocation done while + * holding the struct mutex, we cannot call + * synchronize_rcu_expedited() as it depends on + * workqueues to run but the running workqueue may be + * blocked waiting on us to release struct_mutex. + */ + synchronize_rcu_expedited(); + return freed; } From mboxrd@z Thu Jan 1 00:00:00 1970 From: Andrea Arcangeli Subject: [PATCH 1/5] i915: avoid kernel hang caused by synchronize rcu struct_mutex deadlock Date: Fri, 7 Apr 2017 01:23:43 +0200 Message-ID: <20170406232347.988-2-aarcange@redhat.com> References: <87pogtplxr.fsf@intel.com> <20170406232347.988-1-aarcange@redhat.com> Mime-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: base64 Return-path: In-Reply-To: <20170406232347.988-1-aarcange@redhat.com> List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: intel-gfx-bounces@lists.freedesktop.org Sender: "Intel-gfx" To: Martin Kepplinger , Thorsten Leemhuis , daniel.vetter@intel.com, Dave Airlie , Chris Wilson Cc: intel-gfx@lists.freedesktop.org, linux-kernel@vger.kernel.org, dri-devel@lists.freedesktop.org List-Id: dri-devel@lists.freedesktop.org c3luY2hyb25pemVfcmN1L3N5bmNocm9uaXplX3NjaGVkL3N5bmNocm9uaXplX3JjdV9leHBlZGl0 ZWQoKSB3aWxsCmhhbmcgdW50aWwgaXRzIG93biB3b3JrcXVldWVzIGFyZSBydW4uIFRoZSBpOTE1 IGdlbSB3b3JrcXVldWVzIHdpbGwKd2FpdCBvbiB0aGUgc3RydWN0X211dGV4IHRvIGJlIHJlbGVh c2VkLiBTbyB3ZSBjYW5ub3Qgd2FpdCBmb3IgYQpxdWllc2NlbnQgc3RhdGUgdXNpbmcgdGhvc2Ug cmN1IHByaW1pdGl2ZXMgd2hpbGUgaG9sZGluZyB0aGUKc3RydWN0X211dGV4IG9yIGl0IGNyZWF0 ZXMgYSBjaXJjdWxhciBsb2NrIGRlcGVuZGVuY3kgcmVzdWx0aW5nIGluCmtlcm5lbCBoYW5ncyAo d2hpY2ggaXMgcmVwcm9kdWNpYmxlIGJ1dCBnb2VzIHVuZGV0ZWN0ZWQgYnkgbG9ja2RlcCkuCgpU aGlzIHN0YXJ0ZWQgaW4gY29tbWl0IDNkM2QxOGYwODZjZGRhNzJlZTE4YTQ1NGRiNzBjYTcyYzZl MzI0NmMgYW5kCmxvY2tkZXAgZGlkbid0IGRldGVjdCBpdCBhcHBhcmVudGx5LgoKa3N3YXBkMCAg ICAgICAgIEQgICAgMCAgIDcwMCAgICAgIDIgMHgwMDAwMDAwMApDYWxsIFRyYWNlOgo/IF9fc2No ZWR1bGUrMHgxYTUvMHg2NjAKPyBzY2hlZHVsZSsweDM2LzB4ODAKPyBfc3luY2hyb25pemVfcmN1 X2V4cGVkaXRlZC5jb25zdHByb3AuNjUrMHgyZWYvMHgzMDAKPyB3YWtlX3VwX2JpdCsweDIwLzB4 MjAKPyByY3Vfc3RhbGxfa2lja19rdGhyZWFkcy5wYXJ0LjU0KzB4YzAvMHhjMAo/IHJjdV9leHBf d2FpdF93YWtlKzB4NTMwLzB4NTMwCj8gaTkxNV9nZW1fc2hyaW5rKzB4MzRiLzB4NGIwCj8gaTkx NV9nZW1fc2hyaW5rZXJfc2NhbisweDdjLzB4OTAKPyBpOTE1X2dlbV9zaHJpbmtlcl9zY2FuKzB4 N2MvMHg5MAo/IHNocmlua19zbGFiLnBhcnQuNjEuY29uc3Rwcm9wLjcyKzB4MWMxLzB4M2EwCj8g c2hyaW5rX3pvbmUrMHgxNTQvMHgxNjAKPyBrc3dhcGQrMHg0MGEvMHg3MjAKPyBrdGhyZWFkKzB4 ZjQvMHgxMzAKPyB0cnlfdG9fZnJlZV9wYWdlcysweDQ1MC8weDQ1MAo/IGt0aHJlYWRfY3JlYXRl X29uX25vZGUrMHg0MC8weDQwCj8gcmV0X2Zyb21fZm9yaysweDIzLzB4MzAKcGxhc21hc2hlbGwg ICAgIEQgICAgMCAgNDY1NyAgIDQ2MTQgMHgwMDAwMDAwMApDYWxsIFRyYWNlOgo/IF9fc2NoZWR1 bGUrMHgxYTUvMHg2NjAKPyBzY2hlZHVsZSsweDM2LzB4ODAKPyBzY2hlZHVsZV9wcmVlbXB0X2Rp c2FibGVkKzB4ZS8weDEwCj8gX19tdXRleF9sb2NrLmlzcmEuNCsweDFjOS8weDc5MAo/IGk5MTVf Z2VtX2Nsb3NlX29iamVjdCsweDI2LzB4YzAKPyBpOTE1X2dlbV9jbG9zZV9vYmplY3QrMHgyNi8w eGMwCj8gZHJtX2dlbV9vYmplY3RfcmVsZWFzZV9oYW5kbGUrMHg0OC8weDkwCj8gZHJtX2dlbV9o YW5kbGVfZGVsZXRlKzB4NTAvMHg4MAo/IGRybV9pb2N0bCsweDFmYS8weDQyMAo/IGRybV9nZW1f aGFuZGxlX2NyZWF0ZSsweDQwLzB4NDAKPyBwaXBlX3dyaXRlKzB4MzkxLzB4NDEwCj8gX192ZnNf d3JpdGUrMHhjNi8weDEyMAo/IGRvX3Zmc19pb2N0bCsweDhiLzB4NWQwCj8gU3lTX2lvY3RsKzB4 M2IvMHg3MAo/IGVudHJ5X1NZU0NBTExfNjRfZmFzdHBhdGgrMHgxMy8weDk0Cmt3b3JrZXIvMDow ICAgICBEICAgIDAgMjkxODYgICAgICAyIDB4MDAwMDAwMDAKV29ya3F1ZXVlOiBldmVudHMgX19p OTE1X2dlbV9mcmVlX3dvcmsKQ2FsbCBUcmFjZToKPyBfX3NjaGVkdWxlKzB4MWE1LzB4NjYwCj8g c2NoZWR1bGUrMHgzNi8weDgwCj8gc2NoZWR1bGVfcHJlZW1wdF9kaXNhYmxlZCsweGUvMHgxMAo/ IF9fbXV0ZXhfbG9jay5pc3JhLjQrMHgxYzkvMHg3OTAKPyBkZWxfdGltZXJfc3luYysweDQ0LzB4 NTAKPyB1cGRhdGVfY3VycisweDU3LzB4MTEwCj8gX19pOTE1X2dlbV9mcmVlX29iamVjdHMrMHgz MS8weDMwMAo/IF9faTkxNV9nZW1fZnJlZV9vYmplY3RzKzB4MzEvMHgzMDAKPyBfX2k5MTVfZ2Vt X2ZyZWVfd29yaysweDJkLzB4NDAKPyBwcm9jZXNzX29uZV93b3JrKzB4MTNhLzB4M2IwCj8gd29y a2VyX3RocmVhZCsweDRhLzB4NDYwCj8ga3RocmVhZCsweGY0LzB4MTMwCj8gcHJvY2Vzc19vbmVf d29yaysweDNiMC8weDNiMAo/IGt0aHJlYWRfY3JlYXRlX29uX25vZGUrMHg0MC8weDQwCj8gcmV0 X2Zyb21fZm9yaysweDIzLzB4MzAKClNpZ25lZC1vZmYtYnk6IEFuZHJlYSBBcmNhbmdlbGkgPGFh cmNhbmdlQHJlZGhhdC5jb20+Ci0tLQogZHJpdmVycy9ncHUvZHJtL2k5MTUvaTkxNV9nZW0uYyAg ICAgICAgICB8ICA5ICsrKysrKysrKwogZHJpdmVycy9ncHUvZHJtL2k5MTUvaTkxNV9nZW1fc2hy aW5rZXIuYyB8IDE0ICsrKysrKysrKystLS0tCiAyIGZpbGVzIGNoYW5nZWQsIDE5IGluc2VydGlv bnMoKyksIDQgZGVsZXRpb25zKC0pCgpkaWZmIC0tZ2l0IGEvZHJpdmVycy9ncHUvZHJtL2k5MTUv aTkxNV9nZW0uYyBiL2RyaXZlcnMvZ3B1L2RybS9pOTE1L2k5MTVfZ2VtLmMKaW5kZXggNjdiMWZj NS4uMzk4MjQ4OSAxMDA2NDQKLS0tIGEvZHJpdmVycy9ncHUvZHJtL2k5MTUvaTkxNV9nZW0uYwor KysgYi9kcml2ZXJzL2dwdS9kcm0vaTkxNS9pOTE1X2dlbS5jCkBAIC00NzQyLDYgKzQ3NDIsMTMg QEAgaW50IGk5MTVfZ2VtX2ZyZWV6ZShzdHJ1Y3QgZHJtX2k5MTVfcHJpdmF0ZSAqZGV2X3ByaXYp CiAJaTkxNV9nZW1fc2hyaW5rX2FsbChkZXZfcHJpdik7CiAJbXV0ZXhfdW5sb2NrKCZkZXZfcHJp di0+ZHJtLnN0cnVjdF9tdXRleCk7CiAKKwkvKgorCSAqIENhbm5vdCBjYWxsIHN5bmNocm9uaXpl X3JjdSgpIGluc2lkZSB0aGUgc3RydWN0X211dGV4CisJICogYmVjYXVzZSBpdCBtYXkgYmxvY2sg dW50aWwgd29ya3F1ZXVlcyBjb21wbGV0ZSwgYW5kIHRoZQorCSAqIHJ1bm5pbmcgd29ya3F1ZXVl IG1heSB3YWl0IG9uIHRoZSBzdHJ1Y3RfbXV0ZXguCisJICovCisJc3luY2hyb25pemVfcmN1KCk7 IC8qIHdhaXQgZm9yIG91ciBlYXJsaWVyIFJDVSBkZWxheWVkIHNsYWIgZnJlZXMgKi8KKwogCWlu dGVsX3J1bnRpbWVfcG1fcHV0KGRldl9wcml2KTsKIAogCXJldHVybiAwOwpAQCAtNDc4MSw2ICs0 Nzg4LDggQEAgaW50IGk5MTVfZ2VtX2ZyZWV6ZV9sYXRlKHN0cnVjdCBkcm1faTkxNV9wcml2YXRl ICpkZXZfcHJpdikKIAl9CiAJbXV0ZXhfdW5sb2NrKCZkZXZfcHJpdi0+ZHJtLnN0cnVjdF9tdXRl eCk7CiAKKwlzeW5jaHJvbml6ZV9yY3VfZXhwZWRpdGVkKCk7CisKIAlyZXR1cm4gMDsKIH0KIApk aWZmIC0tZ2l0IGEvZHJpdmVycy9ncHUvZHJtL2k5MTUvaTkxNV9nZW1fc2hyaW5rZXIuYyBiL2Ry aXZlcnMvZ3B1L2RybS9pOTE1L2k5MTVfZ2VtX3Nocmlua2VyLmMKaW5kZXggZDVkMmI0Yy4uZmVh MTQ1NCAxMDA2NDQKLS0tIGEvZHJpdmVycy9ncHUvZHJtL2k5MTUvaTkxNV9nZW1fc2hyaW5rZXIu YworKysgYi9kcml2ZXJzL2dwdS9kcm0vaTkxNS9pOTE1X2dlbV9zaHJpbmtlci5jCkBAIC0yMzUs OSArMjM1LDYgQEAgaTkxNV9nZW1fc2hyaW5rKHN0cnVjdCBkcm1faTkxNV9wcml2YXRlICpkZXZf cHJpdiwKIAlpZiAodW5sb2NrKQogCQltdXRleF91bmxvY2soJmRldl9wcml2LT5kcm0uc3RydWN0 X211dGV4KTsKIAotCS8qIGV4cGVkaXRlIHRoZSBSQ1UgZ3JhY2UgcGVyaW9kIHRvIGZyZWUgc29t ZSByZXF1ZXN0IHNsYWJzICovCi0Jc3luY2hyb25pemVfcmN1X2V4cGVkaXRlZCgpOwotCiAJcmV0 dXJuIGNvdW50OwogfQogCkBAIC0yNjMsNyArMjYwLDYgQEAgdW5zaWduZWQgbG9uZyBpOTE1X2dl bV9zaHJpbmtfYWxsKHN0cnVjdCBkcm1faTkxNV9wcml2YXRlICpkZXZfcHJpdikKIAkJCQlJOTE1 X1NIUklOS19CT1VORCB8CiAJCQkJSTkxNV9TSFJJTktfVU5CT1VORCB8CiAJCQkJSTkxNV9TSFJJ TktfQUNUSVZFKTsKLQlzeW5jaHJvbml6ZV9yY3UoKTsgLyogd2FpdCBmb3Igb3VyIGVhcmxpZXIg UkNVIGRlbGF5ZWQgc2xhYiBmcmVlcyAqLwogCiAJcmV0dXJuIGZyZWVkOwogfQpAQCAtMzI0LDYg KzMyMCwxNiBAQCBpOTE1X2dlbV9zaHJpbmtlcl9zY2FuKHN0cnVjdCBzaHJpbmtlciAqc2hyaW5r ZXIsIHN0cnVjdCBzaHJpbmtfY29udHJvbCAqc2MpCiAJaWYgKHVubG9jaykKIAkJbXV0ZXhfdW5s b2NrKCZkZXYtPnN0cnVjdF9tdXRleCk7CiAKKwlpZiAobGlrZWx5KF9fbXV0ZXhfb3duZXIoJmRl di0+c3RydWN0X211dGV4KSAhPSBjdXJyZW50KSkKKwkJLyoKKwkJICogSWYgcmVjbGFpbSB3YXMg aW52b2tlZCBieSBhbiBhbGxvY2F0aW9uIGRvbmUgd2hpbGUKKwkJICogaG9sZGluZyB0aGUgc3Ry dWN0IG11dGV4LCB3ZSBjYW5ub3QgY2FsbAorCQkgKiBzeW5jaHJvbml6ZV9yY3VfZXhwZWRpdGVk KCkgYXMgaXQgZGVwZW5kcyBvbgorCQkgKiB3b3JrcXVldWVzIHRvIHJ1biBidXQgdGhlIHJ1bm5p bmcgd29ya3F1ZXVlIG1heSBiZQorCQkgKiBibG9ja2VkIHdhaXRpbmcgb24gdXMgdG8gcmVsZWFz ZSBzdHJ1Y3RfbXV0ZXguCisJCSAqLworCQlzeW5jaHJvbml6ZV9yY3VfZXhwZWRpdGVkKCk7CisK IAlyZXR1cm4gZnJlZWQ7CiB9CiAKX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19fX19f X19fX19fX19fX18KSW50ZWwtZ2Z4IG1haWxpbmcgbGlzdApJbnRlbC1nZnhAbGlzdHMuZnJlZWRl c2t0b3Aub3JnCmh0dHBzOi8vbGlzdHMuZnJlZWRlc2t0b3Aub3JnL21haWxtYW4vbGlzdGluZm8v aW50ZWwtZ2Z4Cg==