* [PATCH 1/4] mm/list_lru: use list_lru_walk_one() in list_lru_walk_node()
2018-07-16 11:19 [PATCH 0/4] mm/list_lru: Add list_lru_shrink_walk_irq() and a user Sebastian Andrzej Siewior
@ 2018-07-16 11:19 ` Sebastian Andrzej Siewior
2018-07-16 11:19 ` [PATCH 2/4] mm/list_lru: Move locking from __list_lru_walk_one() to its caller Sebastian Andrzej Siewior
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Sebastian Andrzej Siewior @ 2018-07-16 11:19 UTC (permalink / raw)
To: linux-mm; +Cc: tglx, Vladimir Davydov, Andrew Morton, Sebastian Andrzej Siewior
list_lru_walk_node() invokes __list_lru_walk_one() with -1 as the
memcg_idx parameter. The same can be achieved by list_lru_walk_one() and
passing NULL as memcg argument which then gets converted into -1. This
is a preparation step when the spin_lock() function is lifted to the
caller of __list_lru_walk_one().
Invoke list_lru_walk_one() instead __list_lru_walk_one() when possible.
Reviewed-by: Vladimir Davydov <vdavydov.dev@gmail.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
mm/list_lru.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/mm/list_lru.c b/mm/list_lru.c
index ca6dbcfe4256..344306714636 100644
--- a/mm/list_lru.c
+++ b/mm/list_lru.c
@@ -294,8 +294,8 @@ unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
long isolated = 0;
int memcg_idx;
- isolated += __list_lru_walk_one(lru, nid, -1, isolate, cb_arg,
- nr_to_walk);
+ isolated += list_lru_walk_one(lru, nid, NULL, isolate, cb_arg,
+ nr_to_walk);
if (*nr_to_walk > 0 && list_lru_memcg_aware(lru)) {
for_each_memcg_cache_index(memcg_idx) {
isolated += __list_lru_walk_one(lru, nid, memcg_idx,
--
2.18.0
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 2/4] mm/list_lru: Move locking from __list_lru_walk_one() to its caller
2018-07-16 11:19 [PATCH 0/4] mm/list_lru: Add list_lru_shrink_walk_irq() and a user Sebastian Andrzej Siewior
2018-07-16 11:19 ` [PATCH 1/4] mm/list_lru: use list_lru_walk_one() in list_lru_walk_node() Sebastian Andrzej Siewior
@ 2018-07-16 11:19 ` Sebastian Andrzej Siewior
2018-07-16 11:19 ` [PATCH 3/4] mm/list_lru: Pass struct list_lru_node as an argument __list_lru_walk_one() Sebastian Andrzej Siewior
2018-07-16 11:19 ` [PATCH 4/4] mm/list_lru: Introduce list_lru_shrink_walk_irq() Sebastian Andrzej Siewior
3 siblings, 0 replies; 5+ messages in thread
From: Sebastian Andrzej Siewior @ 2018-07-16 11:19 UTC (permalink / raw)
To: linux-mm; +Cc: tglx, Vladimir Davydov, Andrew Morton, Sebastian Andrzej Siewior
Move the locking inside __list_lru_walk_one() to its caller. This is a
preparation step in order to introduce list_lru_walk_one_irq() which
does spin_lock_irq() instead of spin_lock() for the locking.
Reviewed-by: Vladimir Davydov <vdavydov.dev@gmail.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
mm/list_lru.c | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git a/mm/list_lru.c b/mm/list_lru.c
index 344306714636..3e36e7a239e5 100644
--- a/mm/list_lru.c
+++ b/mm/list_lru.c
@@ -226,7 +226,6 @@ __list_lru_walk_one(struct list_lru *lru, int nid, int memcg_idx,
struct list_head *item, *n;
unsigned long isolated = 0;
- spin_lock(&nlru->lock);
l = list_lru_from_memcg_idx(nlru, memcg_idx);
restart:
list_for_each_safe(item, n, &l->list) {
@@ -272,8 +271,6 @@ __list_lru_walk_one(struct list_lru *lru, int nid, int memcg_idx,
BUG();
}
}
-
- spin_unlock(&nlru->lock);
return isolated;
}
@@ -282,8 +279,14 @@ list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg,
list_lru_walk_cb isolate, void *cb_arg,
unsigned long *nr_to_walk)
{
- return __list_lru_walk_one(lru, nid, memcg_cache_id(memcg),
- isolate, cb_arg, nr_to_walk);
+ struct list_lru_node *nlru = &lru->node[nid];
+ unsigned long ret;
+
+ spin_lock(&nlru->lock);
+ ret = __list_lru_walk_one(lru, nid, memcg_cache_id(memcg),
+ isolate, cb_arg, nr_to_walk);
+ spin_unlock(&nlru->lock);
+ return ret;
}
EXPORT_SYMBOL_GPL(list_lru_walk_one);
@@ -298,8 +301,13 @@ unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
nr_to_walk);
if (*nr_to_walk > 0 && list_lru_memcg_aware(lru)) {
for_each_memcg_cache_index(memcg_idx) {
+ struct list_lru_node *nlru = &lru->node[nid];
+
+ spin_lock(&nlru->lock);
isolated += __list_lru_walk_one(lru, nid, memcg_idx,
isolate, cb_arg, nr_to_walk);
+ spin_unlock(&nlru->lock);
+
if (*nr_to_walk <= 0)
break;
}
--
2.18.0
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 3/4] mm/list_lru: Pass struct list_lru_node as an argument __list_lru_walk_one()
2018-07-16 11:19 [PATCH 0/4] mm/list_lru: Add list_lru_shrink_walk_irq() and a user Sebastian Andrzej Siewior
2018-07-16 11:19 ` [PATCH 1/4] mm/list_lru: use list_lru_walk_one() in list_lru_walk_node() Sebastian Andrzej Siewior
2018-07-16 11:19 ` [PATCH 2/4] mm/list_lru: Move locking from __list_lru_walk_one() to its caller Sebastian Andrzej Siewior
@ 2018-07-16 11:19 ` Sebastian Andrzej Siewior
2018-07-16 11:19 ` [PATCH 4/4] mm/list_lru: Introduce list_lru_shrink_walk_irq() Sebastian Andrzej Siewior
3 siblings, 0 replies; 5+ messages in thread
From: Sebastian Andrzej Siewior @ 2018-07-16 11:19 UTC (permalink / raw)
To: linux-mm; +Cc: tglx, Vladimir Davydov, Andrew Morton, Sebastian Andrzej Siewior
__list_lru_walk_one() is invoked with struct list_lru *lru, int nid as
the first two argument. Those two are only used to retrieve struct
list_lru_node. Since this is already done by the caller of the function
for the locking, we can pass struct list_lru_node directly and avoid the
dance around it.
Reviewed-by: Vladimir Davydov <vdavydov.dev@gmail.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
mm/list_lru.c | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/mm/list_lru.c b/mm/list_lru.c
index 3e36e7a239e5..7b7a737f0963 100644
--- a/mm/list_lru.c
+++ b/mm/list_lru.c
@@ -216,12 +216,11 @@ unsigned long list_lru_count_node(struct list_lru *lru, int nid)
EXPORT_SYMBOL_GPL(list_lru_count_node);
static unsigned long
-__list_lru_walk_one(struct list_lru *lru, int nid, int memcg_idx,
+__list_lru_walk_one(struct list_lru_node *nlru, int memcg_idx,
list_lru_walk_cb isolate, void *cb_arg,
unsigned long *nr_to_walk)
{
- struct list_lru_node *nlru = &lru->node[nid];
struct list_lru_one *l;
struct list_head *item, *n;
unsigned long isolated = 0;
@@ -283,8 +282,8 @@ list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg,
unsigned long ret;
spin_lock(&nlru->lock);
- ret = __list_lru_walk_one(lru, nid, memcg_cache_id(memcg),
- isolate, cb_arg, nr_to_walk);
+ ret = __list_lru_walk_one(nlru, memcg_cache_id(memcg), isolate, cb_arg,
+ nr_to_walk);
spin_unlock(&nlru->lock);
return ret;
}
@@ -304,8 +303,9 @@ unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
struct list_lru_node *nlru = &lru->node[nid];
spin_lock(&nlru->lock);
- isolated += __list_lru_walk_one(lru, nid, memcg_idx,
- isolate, cb_arg, nr_to_walk);
+ isolated += __list_lru_walk_one(nlru, memcg_idx,
+ isolate, cb_arg,
+ nr_to_walk);
spin_unlock(&nlru->lock);
if (*nr_to_walk <= 0)
--
2.18.0
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 4/4] mm/list_lru: Introduce list_lru_shrink_walk_irq()
2018-07-16 11:19 [PATCH 0/4] mm/list_lru: Add list_lru_shrink_walk_irq() and a user Sebastian Andrzej Siewior
` (2 preceding siblings ...)
2018-07-16 11:19 ` [PATCH 3/4] mm/list_lru: Pass struct list_lru_node as an argument __list_lru_walk_one() Sebastian Andrzej Siewior
@ 2018-07-16 11:19 ` Sebastian Andrzej Siewior
3 siblings, 0 replies; 5+ messages in thread
From: Sebastian Andrzej Siewior @ 2018-07-16 11:19 UTC (permalink / raw)
To: linux-mm; +Cc: tglx, Vladimir Davydov, Andrew Morton, Sebastian Andrzej Siewior
Provide list_lru_shrink_walk_irq() and let it behave like
list_lru_walk_one() except that it locks the spinlock with
spin_lock_irq(). This is used by scan_shadow_nodes() because its lock
nests within the i_pages lock which is acquired with IRQ.
This change allows to use proper locking promitives instead hand crafted
lock_irq_disable() plus spin_lock().
There is no EXPORT_SYMBOL provided because the current user is in-KERNEL
only.
Add list_lru_shrink_walk_irq() which acquires the spinlock with the
proper locking primitives.
Reviewed-by: Vladimir Davydov <vdavydov.dev@gmail.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
include/linux/list_lru.h | 25 +++++++++++++++++++++++++
mm/list_lru.c | 15 +++++++++++++++
mm/workingset.c | 8 ++------
3 files changed, 42 insertions(+), 6 deletions(-)
diff --git a/include/linux/list_lru.h b/include/linux/list_lru.h
index d9c16f2f2f00..aa5efd9351eb 100644
--- a/include/linux/list_lru.h
+++ b/include/linux/list_lru.h
@@ -166,6 +166,23 @@ unsigned long list_lru_walk_one(struct list_lru *lru,
int nid, struct mem_cgroup *memcg,
list_lru_walk_cb isolate, void *cb_arg,
unsigned long *nr_to_walk);
+/**
+ * list_lru_walk_one_irq: walk a list_lru, isolating and disposing freeable items.
+ * @lru: the lru pointer.
+ * @nid: the node id to scan from.
+ * @memcg: the cgroup to scan from.
+ * @isolate: callback function that is resposible for deciding what to do with
+ * the item currently being scanned
+ * @cb_arg: opaque type that will be passed to @isolate
+ * @nr_to_walk: how many items to scan.
+ *
+ * Same as @list_lru_walk_one except that the spinlock is acquired with
+ * spin_lock_irq().
+ */
+unsigned long list_lru_walk_one_irq(struct list_lru *lru,
+ int nid, struct mem_cgroup *memcg,
+ list_lru_walk_cb isolate, void *cb_arg,
+ unsigned long *nr_to_walk);
unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
list_lru_walk_cb isolate, void *cb_arg,
unsigned long *nr_to_walk);
@@ -178,6 +195,14 @@ list_lru_shrink_walk(struct list_lru *lru, struct shrink_control *sc,
&sc->nr_to_scan);
}
+static inline unsigned long
+list_lru_shrink_walk_irq(struct list_lru *lru, struct shrink_control *sc,
+ list_lru_walk_cb isolate, void *cb_arg)
+{
+ return list_lru_walk_one_irq(lru, sc->nid, sc->memcg, isolate, cb_arg,
+ &sc->nr_to_scan);
+}
+
static inline unsigned long
list_lru_walk(struct list_lru *lru, list_lru_walk_cb isolate,
void *cb_arg, unsigned long nr_to_walk)
diff --git a/mm/list_lru.c b/mm/list_lru.c
index 7b7a737f0963..89349a0276de 100644
--- a/mm/list_lru.c
+++ b/mm/list_lru.c
@@ -289,6 +289,21 @@ list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg,
}
EXPORT_SYMBOL_GPL(list_lru_walk_one);
+unsigned long
+list_lru_walk_one_irq(struct list_lru *lru, int nid, struct mem_cgroup *memcg,
+ list_lru_walk_cb isolate, void *cb_arg,
+ unsigned long *nr_to_walk)
+{
+ struct list_lru_node *nlru = &lru->node[nid];
+ unsigned long ret;
+
+ spin_lock_irq(&nlru->lock);
+ ret = __list_lru_walk_one(nlru, memcg_cache_id(memcg), isolate, cb_arg,
+ nr_to_walk);
+ spin_unlock_irq(&nlru->lock);
+ return ret;
+}
+
unsigned long list_lru_walk_node(struct list_lru *lru, int nid,
list_lru_walk_cb isolate, void *cb_arg,
unsigned long *nr_to_walk)
diff --git a/mm/workingset.c b/mm/workingset.c
index 06b45147e892..0b4f471d07ba 100644
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -501,13 +501,9 @@ static enum lru_status shadow_lru_isolate(struct list_head *item,
static unsigned long scan_shadow_nodes(struct shrinker *shrinker,
struct shrink_control *sc)
{
- unsigned long ret;
-
/* list_lru lock nests inside the IRQ-safe i_pages lock */
- local_irq_disable();
- ret = list_lru_shrink_walk(&shadow_nodes, sc, shadow_lru_isolate, NULL);
- local_irq_enable();
- return ret;
+ return list_lru_shrink_walk_irq(&shadow_nodes, sc, shadow_lru_isolate,
+ NULL);
}
static struct shrinker workingset_shadow_shrinker = {
--
2.18.0
^ permalink raw reply related [flat|nested] 5+ messages in thread