From: Rob Clark <robdclark@gmail.com> To: dri-devel@lists.freedesktop.org Cc: Rob Clark <robdclark@chromium.org>, Rob Clark <robdclark@gmail.com>, Sean Paul <sean@poorly.run>, David Airlie <airlied@linux.ie>, Daniel Vetter <daniel@ffwll.ch>, linux-arm-msm@vger.kernel.org (open list:DRM DRIVER FOR MSM ADRENO GPU), freedreno@lists.freedesktop.org (open list:DRM DRIVER FOR MSM ADRENO GPU), linux-kernel@vger.kernel.org (open list) Subject: [PATCH 2/4] drm/msm: Avoid mutex in shrinker_count() Date: Wed, 31 Mar 2021 15:16:27 -0700 [thread overview] Message-ID: <20210331221630.488498-3-robdclark@gmail.com> (raw) In-Reply-To: <20210331221630.488498-1-robdclark@gmail.com> From: Rob Clark <robdclark@chromium.org> When the system is under heavy memory pressure, we can end up with lots of concurrent calls into the shrinker. Keeping a running tab on what we can shrink avoids grabbing a lock in shrinker->count(), and avoids shrinker->scan() getting called when not profitable. Also, we can keep purged objects in their own list to avoid re-traversing them to help cut down time in the critical section further. Signed-off-by: Rob Clark <robdclark@chromium.org> --- drivers/gpu/drm/msm/msm_drv.c | 1 + drivers/gpu/drm/msm/msm_drv.h | 2 ++ drivers/gpu/drm/msm/msm_gem.c | 16 +++++++++++-- drivers/gpu/drm/msm/msm_gem.h | 32 ++++++++++++++++++++++++++ drivers/gpu/drm/msm/msm_gem_shrinker.c | 17 +------------- 5 files changed, 50 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index 4f9fa0189a07..3462b0ea14c6 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -476,6 +476,7 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) INIT_LIST_HEAD(&priv->inactive_willneed); INIT_LIST_HEAD(&priv->inactive_dontneed); + INIT_LIST_HEAD(&priv->inactive_purged); mutex_init(&priv->mm_lock); /* Teach lockdep about lock ordering wrt. shrinker: */ diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h index a1264cfcac5e..3ead5755f695 100644 --- a/drivers/gpu/drm/msm/msm_drv.h +++ b/drivers/gpu/drm/msm/msm_drv.h @@ -188,6 +188,8 @@ struct msm_drm_private { */ struct list_head inactive_willneed; /* inactive + !shrinkable */ struct list_head inactive_dontneed; /* inactive + shrinkable */ + struct list_head inactive_purged; /* inactive + purged */ + int shrinkable_count; /* write access under mm_lock */ struct mutex mm_lock; struct workqueue_struct *wq; diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index 9d10739c4eb2..74a92eedc992 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -719,6 +719,7 @@ void msm_gem_purge(struct drm_gem_object *obj) put_iova_vmas(obj); msm_obj->madv = __MSM_MADV_PURGED; + mark_unpurgable(msm_obj); drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping); drm_gem_free_mmap_offset(obj); @@ -790,6 +791,7 @@ void msm_gem_active_get(struct drm_gem_object *obj, struct msm_gpu *gpu) might_sleep(); WARN_ON(!msm_gem_is_locked(obj)); WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED); + WARN_ON(msm_obj->dontneed); if (msm_obj->active_count++ == 0) { mutex_lock(&priv->mm_lock); @@ -818,11 +820,19 @@ static void update_inactive(struct msm_gem_object *msm_obj) mutex_lock(&priv->mm_lock); WARN_ON(msm_obj->active_count != 0); + if (msm_obj->dontneed) + mark_unpurgable(msm_obj); + list_del_init(&msm_obj->mm_list); - if (msm_obj->madv == MSM_MADV_WILLNEED) + if (msm_obj->madv == MSM_MADV_WILLNEED) { list_add_tail(&msm_obj->mm_list, &priv->inactive_willneed); - else + } else if (msm_obj->madv == MSM_MADV_DONTNEED) { list_add_tail(&msm_obj->mm_list, &priv->inactive_dontneed); + mark_purgable(msm_obj); + } else { + WARN_ON(msm_obj->madv != __MSM_MADV_PURGED); + list_add_tail(&msm_obj->mm_list, &priv->inactive_purged); + } mutex_unlock(&priv->mm_lock); } @@ -971,6 +981,8 @@ void msm_gem_free_object(struct drm_gem_object *obj) struct msm_drm_private *priv = dev->dev_private; mutex_lock(&priv->mm_lock); + if (msm_obj->dontneed) + mark_unpurgable(msm_obj); list_del(&msm_obj->mm_list); mutex_unlock(&priv->mm_lock); diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h index 7a9107cf1818..0feabae75d3d 100644 --- a/drivers/gpu/drm/msm/msm_gem.h +++ b/drivers/gpu/drm/msm/msm_gem.h @@ -50,6 +50,11 @@ struct msm_gem_object { */ uint8_t madv; + /** + * Is object on inactive_dontneed list (ie. counted in priv->shrinkable_count)? + */ + bool dontneed : 1; + /** * count of active vmap'ing */ @@ -198,6 +203,33 @@ static inline bool is_vunmapable(struct msm_gem_object *msm_obj) return (msm_obj->vmap_count == 0) && msm_obj->vaddr; } +static inline void mark_purgable(struct msm_gem_object *msm_obj) +{ + struct msm_drm_private *priv = msm_obj->base.dev->dev_private; + + WARN_ON(!mutex_is_locked(&priv->mm_lock)); + + if (WARN_ON(msm_obj->dontneed)) + return; + + priv->shrinkable_count += msm_obj->base.size >> PAGE_SHIFT; + msm_obj->dontneed = true; +} + +static inline void mark_unpurgable(struct msm_gem_object *msm_obj) +{ + struct msm_drm_private *priv = msm_obj->base.dev->dev_private; + + WARN_ON(!mutex_is_locked(&priv->mm_lock)); + + if (WARN_ON(!msm_obj->dontneed)) + return; + + priv->shrinkable_count -= msm_obj->base.size >> PAGE_SHIFT; + WARN_ON(priv->shrinkable_count < 0); + msm_obj->dontneed = false; +} + void msm_gem_purge(struct drm_gem_object *obj); void msm_gem_vunmap(struct drm_gem_object *obj); diff --git a/drivers/gpu/drm/msm/msm_gem_shrinker.c b/drivers/gpu/drm/msm/msm_gem_shrinker.c index 9d5248be746f..7db8375f2430 100644 --- a/drivers/gpu/drm/msm/msm_gem_shrinker.c +++ b/drivers/gpu/drm/msm/msm_gem_shrinker.c @@ -14,22 +14,7 @@ msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) { struct msm_drm_private *priv = container_of(shrinker, struct msm_drm_private, shrinker); - struct msm_gem_object *msm_obj; - unsigned long count = 0; - - mutex_lock(&priv->mm_lock); - - list_for_each_entry(msm_obj, &priv->inactive_dontneed, mm_list) { - if (!msm_gem_trylock(&msm_obj->base)) - continue; - if (is_purgeable(msm_obj)) - count += msm_obj->base.size >> PAGE_SHIFT; - msm_gem_unlock(&msm_obj->base); - } - - mutex_unlock(&priv->mm_lock); - - return count; + return priv->shrinkable_count; } static unsigned long -- 2.30.2
WARNING: multiple messages have this Message-ID (diff)
From: Rob Clark <robdclark@gmail.com> To: dri-devel@lists.freedesktop.org Cc: Rob Clark <robdclark@chromium.org>, "open list:DRM DRIVER FOR MSM ADRENO GPU" <freedreno@lists.freedesktop.org>, David Airlie <airlied@linux.ie>, "open list:DRM DRIVER FOR MSM ADRENO GPU" <linux-arm-msm@vger.kernel.org>, open list <linux-kernel@vger.kernel.org>, Sean Paul <sean@poorly.run> Subject: [PATCH 2/4] drm/msm: Avoid mutex in shrinker_count() Date: Wed, 31 Mar 2021 15:16:27 -0700 [thread overview] Message-ID: <20210331221630.488498-3-robdclark@gmail.com> (raw) In-Reply-To: <20210331221630.488498-1-robdclark@gmail.com> From: Rob Clark <robdclark@chromium.org> When the system is under heavy memory pressure, we can end up with lots of concurrent calls into the shrinker. Keeping a running tab on what we can shrink avoids grabbing a lock in shrinker->count(), and avoids shrinker->scan() getting called when not profitable. Also, we can keep purged objects in their own list to avoid re-traversing them to help cut down time in the critical section further. Signed-off-by: Rob Clark <robdclark@chromium.org> --- drivers/gpu/drm/msm/msm_drv.c | 1 + drivers/gpu/drm/msm/msm_drv.h | 2 ++ drivers/gpu/drm/msm/msm_gem.c | 16 +++++++++++-- drivers/gpu/drm/msm/msm_gem.h | 32 ++++++++++++++++++++++++++ drivers/gpu/drm/msm/msm_gem_shrinker.c | 17 +------------- 5 files changed, 50 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index 4f9fa0189a07..3462b0ea14c6 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -476,6 +476,7 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv) INIT_LIST_HEAD(&priv->inactive_willneed); INIT_LIST_HEAD(&priv->inactive_dontneed); + INIT_LIST_HEAD(&priv->inactive_purged); mutex_init(&priv->mm_lock); /* Teach lockdep about lock ordering wrt. shrinker: */ diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h index a1264cfcac5e..3ead5755f695 100644 --- a/drivers/gpu/drm/msm/msm_drv.h +++ b/drivers/gpu/drm/msm/msm_drv.h @@ -188,6 +188,8 @@ struct msm_drm_private { */ struct list_head inactive_willneed; /* inactive + !shrinkable */ struct list_head inactive_dontneed; /* inactive + shrinkable */ + struct list_head inactive_purged; /* inactive + purged */ + int shrinkable_count; /* write access under mm_lock */ struct mutex mm_lock; struct workqueue_struct *wq; diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index 9d10739c4eb2..74a92eedc992 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -719,6 +719,7 @@ void msm_gem_purge(struct drm_gem_object *obj) put_iova_vmas(obj); msm_obj->madv = __MSM_MADV_PURGED; + mark_unpurgable(msm_obj); drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping); drm_gem_free_mmap_offset(obj); @@ -790,6 +791,7 @@ void msm_gem_active_get(struct drm_gem_object *obj, struct msm_gpu *gpu) might_sleep(); WARN_ON(!msm_gem_is_locked(obj)); WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED); + WARN_ON(msm_obj->dontneed); if (msm_obj->active_count++ == 0) { mutex_lock(&priv->mm_lock); @@ -818,11 +820,19 @@ static void update_inactive(struct msm_gem_object *msm_obj) mutex_lock(&priv->mm_lock); WARN_ON(msm_obj->active_count != 0); + if (msm_obj->dontneed) + mark_unpurgable(msm_obj); + list_del_init(&msm_obj->mm_list); - if (msm_obj->madv == MSM_MADV_WILLNEED) + if (msm_obj->madv == MSM_MADV_WILLNEED) { list_add_tail(&msm_obj->mm_list, &priv->inactive_willneed); - else + } else if (msm_obj->madv == MSM_MADV_DONTNEED) { list_add_tail(&msm_obj->mm_list, &priv->inactive_dontneed); + mark_purgable(msm_obj); + } else { + WARN_ON(msm_obj->madv != __MSM_MADV_PURGED); + list_add_tail(&msm_obj->mm_list, &priv->inactive_purged); + } mutex_unlock(&priv->mm_lock); } @@ -971,6 +981,8 @@ void msm_gem_free_object(struct drm_gem_object *obj) struct msm_drm_private *priv = dev->dev_private; mutex_lock(&priv->mm_lock); + if (msm_obj->dontneed) + mark_unpurgable(msm_obj); list_del(&msm_obj->mm_list); mutex_unlock(&priv->mm_lock); diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h index 7a9107cf1818..0feabae75d3d 100644 --- a/drivers/gpu/drm/msm/msm_gem.h +++ b/drivers/gpu/drm/msm/msm_gem.h @@ -50,6 +50,11 @@ struct msm_gem_object { */ uint8_t madv; + /** + * Is object on inactive_dontneed list (ie. counted in priv->shrinkable_count)? + */ + bool dontneed : 1; + /** * count of active vmap'ing */ @@ -198,6 +203,33 @@ static inline bool is_vunmapable(struct msm_gem_object *msm_obj) return (msm_obj->vmap_count == 0) && msm_obj->vaddr; } +static inline void mark_purgable(struct msm_gem_object *msm_obj) +{ + struct msm_drm_private *priv = msm_obj->base.dev->dev_private; + + WARN_ON(!mutex_is_locked(&priv->mm_lock)); + + if (WARN_ON(msm_obj->dontneed)) + return; + + priv->shrinkable_count += msm_obj->base.size >> PAGE_SHIFT; + msm_obj->dontneed = true; +} + +static inline void mark_unpurgable(struct msm_gem_object *msm_obj) +{ + struct msm_drm_private *priv = msm_obj->base.dev->dev_private; + + WARN_ON(!mutex_is_locked(&priv->mm_lock)); + + if (WARN_ON(!msm_obj->dontneed)) + return; + + priv->shrinkable_count -= msm_obj->base.size >> PAGE_SHIFT; + WARN_ON(priv->shrinkable_count < 0); + msm_obj->dontneed = false; +} + void msm_gem_purge(struct drm_gem_object *obj); void msm_gem_vunmap(struct drm_gem_object *obj); diff --git a/drivers/gpu/drm/msm/msm_gem_shrinker.c b/drivers/gpu/drm/msm/msm_gem_shrinker.c index 9d5248be746f..7db8375f2430 100644 --- a/drivers/gpu/drm/msm/msm_gem_shrinker.c +++ b/drivers/gpu/drm/msm/msm_gem_shrinker.c @@ -14,22 +14,7 @@ msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) { struct msm_drm_private *priv = container_of(shrinker, struct msm_drm_private, shrinker); - struct msm_gem_object *msm_obj; - unsigned long count = 0; - - mutex_lock(&priv->mm_lock); - - list_for_each_entry(msm_obj, &priv->inactive_dontneed, mm_list) { - if (!msm_gem_trylock(&msm_obj->base)) - continue; - if (is_purgeable(msm_obj)) - count += msm_obj->base.size >> PAGE_SHIFT; - msm_gem_unlock(&msm_obj->base); - } - - mutex_unlock(&priv->mm_lock); - - return count; + return priv->shrinkable_count; } static unsigned long -- 2.30.2 _______________________________________________ dri-devel mailing list dri-devel@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/dri-devel
next prev parent reply other threads:[~2021-03-31 22:14 UTC|newest] Thread overview: 47+ messages / expand[flat|nested] mbox.gz Atom feed top 2021-03-31 22:16 [PATCH 0/4] drm/msm: Shrinker (and related) fixes Rob Clark 2021-03-31 22:16 ` Rob Clark 2021-03-31 22:16 ` [PATCH 1/4] drm/msm: Remove unused freed llist node Rob Clark 2021-03-31 22:16 ` Rob Clark 2021-03-31 23:46 ` Doug Anderson 2021-03-31 23:46 ` Doug Anderson 2021-03-31 22:16 ` Rob Clark [this message] 2021-03-31 22:16 ` [PATCH 2/4] drm/msm: Avoid mutex in shrinker_count() Rob Clark 2021-03-31 22:44 ` Doug Anderson 2021-03-31 22:44 ` Doug Anderson 2021-03-31 23:26 ` Rob Clark 2021-03-31 23:26 ` Rob Clark 2021-03-31 23:39 ` Doug Anderson 2021-03-31 23:39 ` Doug Anderson 2021-04-01 0:17 ` Rob Clark 2021-04-01 0:17 ` Rob Clark 2021-03-31 22:16 ` [PATCH 3/4] drm/msm: Fix debugfs deadlock Rob Clark 2021-03-31 22:16 ` Rob Clark 2021-03-31 23:13 ` Doug Anderson 2021-03-31 23:13 ` Doug Anderson 2021-03-31 23:31 ` Rob Clark 2021-03-31 23:31 ` Rob Clark 2021-03-31 22:16 ` [PATCH 4/4] drm/msm: Improved debugfs gem stats Rob Clark 2021-03-31 22:16 ` Rob Clark 2021-03-31 23:33 ` Doug Anderson 2021-03-31 23:33 ` Doug Anderson 2021-03-31 23:46 ` [PATCH 0/4] drm/msm: Shrinker (and related) fixes Doug Anderson 2021-03-31 23:46 ` Doug Anderson 2021-04-01 1:27 ` [PATCH v2 " Rob Clark 2021-04-01 1:27 ` Rob Clark 2021-04-01 1:27 ` [PATCH v2 1/4] drm/msm: Remove unused freed llist node Rob Clark 2021-04-01 1:27 ` Rob Clark 2021-04-01 15:34 ` Doug Anderson 2021-04-01 15:34 ` Doug Anderson 2021-05-26 19:03 ` patchwork-bot+linux-arm-msm 2021-04-01 1:27 ` [PATCH v2 2/4] drm/msm: Avoid mutex in shrinker_count() Rob Clark 2021-04-01 1:27 ` Rob Clark 2021-04-01 15:34 ` Doug Anderson 2021-04-01 15:34 ` Doug Anderson 2021-04-01 17:37 ` Rob Clark 2021-04-01 17:37 ` Rob Clark 2021-04-01 1:27 ` [PATCH v2 3/4] drm/msm: Fix debugfs deadlock Rob Clark 2021-04-01 1:27 ` Rob Clark 2021-04-01 15:34 ` Doug Anderson 2021-04-01 15:34 ` Doug Anderson 2021-04-01 1:27 ` [PATCH v2 4/4] drm/msm: Improved debugfs gem stats Rob Clark 2021-04-01 1:27 ` Rob Clark
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20210331221630.488498-3-robdclark@gmail.com \ --to=robdclark@gmail.com \ --cc=airlied@linux.ie \ --cc=daniel@ffwll.ch \ --cc=dri-devel@lists.freedesktop.org \ --cc=freedreno@lists.freedesktop.org \ --cc=linux-arm-msm@vger.kernel.org \ --cc=linux-kernel@vger.kernel.org \ --cc=robdclark@chromium.org \ --cc=sean@poorly.run \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.