* [PATCH 1/2] kvfree_rcu: Release a page cache under memory pressure
@ 2021-03-08 19:49 Uladzislau Rezki (Sony)
2021-03-08 19:49 ` [PATCH 2/2] kvfree_rcu: convert a page cache to lock-free variant Uladzislau Rezki (Sony)
` (3 more replies)
0 siblings, 4 replies; 9+ messages in thread
From: Uladzislau Rezki (Sony) @ 2021-03-08 19:49 UTC (permalink / raw)
To: LKML, RCU, Paul E . McKenney
Cc: Michal Hocko, Andrew Morton, Daniel Axtens, Frederic Weisbecker,
Neeraj Upadhyay, Joel Fernandes, Peter Zijlstra, Thomas Gleixner,
Theodore Y . Ts'o, Sebastian Andrzej Siewior,
Uladzislau Rezki, Oleksiy Avramchenko, Zhang Qiang
From: Zhang Qiang <qiang.zhang@windriver.com>
Add a drain_page_cache() function to drain a per-cpu page cache.
The reason behind of it is a system can run into a low memory
condition, in that case a page shrinker can ask for its users
to free their caches in order to get extra memory available for
other needs in a system.
When a system hits such condition, a page cache is drained for
all CPUs in a system. Apart of that a page cache work is delayed
with 5 seconds interval until a memory pressure disappears.
Co-developed-by: Uladzislau Rezki (Sony) <urezki@gmail.com>
Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com>
Signed-off-by: Zqiang <qiang.zhang@windriver.com>
---
kernel/rcu/tree.c | 59 ++++++++++++++++++++++++++++++++++++++++-------
1 file changed, 51 insertions(+), 8 deletions(-)
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 2c9cf4df942c..9c8cfb01e9a6 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -3163,7 +3163,7 @@ struct kfree_rcu_cpu {
bool initialized;
int count;
- struct work_struct page_cache_work;
+ struct delayed_work page_cache_work;
atomic_t work_in_progress;
struct hrtimer hrtimer;
@@ -3175,6 +3175,17 @@ static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc) = {
.lock = __RAW_SPIN_LOCK_UNLOCKED(krc.lock),
};
+// A page shrinker can ask for freeing extra pages
+// to get them available for other needs in a system.
+// Usually it happens under low memory condition, in
+// that case hold on a bit with page cache filling.
+static bool backoff_page_cache_fill;
+
+// 5 seconds delay. That is long enough to reduce
+// an interfering and racing with a shrinker where
+// the cache is drained.
+#define PAGE_CACHE_FILL_DELAY (5 * HZ)
+
static __always_inline void
debug_rcu_bhead_unqueue(struct kvfree_rcu_bulk_data *bhead)
{
@@ -3229,6 +3240,26 @@ put_cached_bnode(struct kfree_rcu_cpu *krcp,
}
+static int
+drain_page_cache(struct kfree_rcu_cpu *krcp)
+{
+ unsigned long flags;
+ struct llist_node *page_list, *pos, *n;
+ int freed = 0;
+
+ raw_spin_lock_irqsave(&krcp->lock, flags);
+ page_list = llist_del_all(&krcp->bkvcache);
+ krcp->nr_bkv_objs = 0;
+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
+
+ llist_for_each_safe(pos, n, page_list) {
+ free_page((unsigned long)pos);
+ freed++;
+ }
+
+ return freed;
+}
+
/*
* This function is invoked in workqueue context after a grace period.
* It frees all the objects queued on ->bhead_free or ->head_free.
@@ -3419,7 +3450,7 @@ schedule_page_work_fn(struct hrtimer *t)
struct kfree_rcu_cpu *krcp =
container_of(t, struct kfree_rcu_cpu, hrtimer);
- queue_work(system_highpri_wq, &krcp->page_cache_work);
+ queue_delayed_work(system_highpri_wq, &krcp->page_cache_work, 0);
return HRTIMER_NORESTART;
}
@@ -3428,7 +3459,7 @@ static void fill_page_cache_func(struct work_struct *work)
struct kvfree_rcu_bulk_data *bnode;
struct kfree_rcu_cpu *krcp =
container_of(work, struct kfree_rcu_cpu,
- page_cache_work);
+ page_cache_work.work);
unsigned long flags;
bool pushed;
int i;
@@ -3457,10 +3488,14 @@ run_page_cache_worker(struct kfree_rcu_cpu *krcp)
{
if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
!atomic_xchg(&krcp->work_in_progress, 1)) {
- hrtimer_init(&krcp->hrtimer, CLOCK_MONOTONIC,
- HRTIMER_MODE_REL);
- krcp->hrtimer.function = schedule_page_work_fn;
- hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);
+ if (xchg(&backoff_page_cache_fill, false)) {
+ queue_delayed_work(system_wq,
+ &krcp->page_cache_work, PAGE_CACHE_FILL_DELAY);
+ } else {
+ hrtimer_init(&krcp->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ krcp->hrtimer.function = schedule_page_work_fn;
+ hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);
+ }
}
}
@@ -3612,14 +3647,20 @@ kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
{
int cpu;
unsigned long count = 0;
+ unsigned long flags;
/* Snapshot count of all CPUs */
for_each_possible_cpu(cpu) {
struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
count += READ_ONCE(krcp->count);
+
+ raw_spin_lock_irqsave(&krcp->lock, flags);
+ count += krcp->nr_bkv_objs;
+ raw_spin_unlock_irqrestore(&krcp->lock, flags);
}
+ WRITE_ONCE(backoff_page_cache_fill, true);
return count;
}
@@ -3634,6 +3675,8 @@ kfree_rcu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
count = krcp->count;
+ count += drain_page_cache(krcp);
+
raw_spin_lock_irqsave(&krcp->lock, flags);
if (krcp->monitor_todo)
kfree_rcu_drain_unlock(krcp, flags);
@@ -4608,7 +4651,7 @@ static void __init kfree_rcu_batch_init(void)
}
INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
- INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
+ INIT_DELAYED_WORK(&krcp->page_cache_work, fill_page_cache_func);
krcp->initialized = true;
}
if (register_shrinker(&kfree_rcu_shrinker))
--
2.20.1
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 2/2] kvfree_rcu: convert a page cache to lock-free variant
2021-03-08 19:49 [PATCH 1/2] kvfree_rcu: Release a page cache under memory pressure Uladzislau Rezki (Sony)
@ 2021-03-08 19:49 ` Uladzislau Rezki (Sony)
[not found] ` <CA+KHdyX-h2KHEEvZmbeLVdjBzW7o37_GvQrGj55-d4i-+HLboQ@mail.gmail.com>
2021-03-08 23:37 ` kernel test robot
` (2 subsequent siblings)
3 siblings, 1 reply; 9+ messages in thread
From: Uladzislau Rezki (Sony) @ 2021-03-08 19:49 UTC (permalink / raw)
To: LKML, RCU, Paul E . McKenney
Cc: Michal Hocko, Andrew Morton, Daniel Axtens, Frederic Weisbecker,
Neeraj Upadhyay, Joel Fernandes, Peter Zijlstra, Thomas Gleixner,
Theodore Y . Ts'o, Sebastian Andrzej Siewior,
Uladzislau Rezki, Oleksiy Avramchenko
Implement an access to the page cache as lock-free variant. This
is done because there are extra places where an access is required,
therefore making it lock-less will remove any lock contention.
For example we have a shrinker path as well as a reclaim kthread.
In both cases a current CPU can access to a remote per-cpu page
cache that would require taking a lock to protect it.
An "rcuscale" performance test suite can detect it and shows some
slight improvements:
../kvm.sh --memory 16G --torture rcuscale --allcpus --duration 10 \
--kconfig CONFIG_NR_CPUS=64 --bootargs "rcuscale.kfree_rcu_test=1 \
rcuscale.kfree_nthreads=16 rcuscale.holdoff=20 rcuscale.kfree_loops=10000 \
rcuscale.kfree_rcu_test_double=1 torture.disable_onoff_at_boot" --trust-make
100 iterations, checking total time taken by all kfree'ers in ns.:
default: AVG: 10968415107.5 MIN: 10668412500 MAX: 11312145160
patch: AVG: 10787596486.1 MIN: 10397559880 MAX: 11214901050
Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com>
---
kernel/rcu/tree.c | 91 +++++++++++++++++++++++++++++------------------
1 file changed, 56 insertions(+), 35 deletions(-)
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 9c8cfb01e9a6..4f04664d5ac0 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -3167,8 +3167,9 @@ struct kfree_rcu_cpu {
atomic_t work_in_progress;
struct hrtimer hrtimer;
+ // lock-free cache.
struct llist_head bkvcache;
- int nr_bkv_objs;
+ atomic_t nr_bkv_objs;
};
static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc) = {
@@ -3215,49 +3216,79 @@ krc_this_cpu_unlock(struct kfree_rcu_cpu *krcp, unsigned long flags)
raw_spin_unlock_irqrestore(&krcp->lock, flags);
}
+/*
+ * Increment 'v', if 'v' is below 'thresh'. Returns true if we
+ * succeeded, false if 'v' + 1 would be bigger than 'thresh'.
+ *
+ * Decrement 'v' if 'v' is upper 'thresh'. Returns true if we
+ * succeeded, false if 'v' - 1 would be smaller than 'thresh'.
+ */
+static inline bool
+atomic_test_inc_dec(atomic_t *v, unsigned int thresh, bool inc)
+{
+ unsigned int cur = atomic_read(v);
+ unsigned int old;
+
+ for (;;) {
+ if (inc) {
+ if (cur >= thresh)
+ return false;
+ } else {
+ if (cur <= thresh)
+ return false;
+ }
+
+ old = atomic_cmpxchg(v, cur, inc ? (cur + 1):(cur - 1));
+ if (old == cur)
+ break;
+
+ cur = old;
+ }
+
+ return true;
+}
+
static inline struct kvfree_rcu_bulk_data *
get_cached_bnode(struct kfree_rcu_cpu *krcp)
{
- if (!krcp->nr_bkv_objs)
- return NULL;
+ struct kvfree_rcu_bulk_data *bnode = NULL;
- krcp->nr_bkv_objs--;
- return (struct kvfree_rcu_bulk_data *)
- llist_del_first(&krcp->bkvcache);
+ if (atomic_test_inc_dec(&krcp->nr_bkv_objs, 0, false))
+ bnode = (struct kvfree_rcu_bulk_data *)
+ llist_del_first(&krcp->bkvcache);
+
+ return bnode;
}
static inline bool
put_cached_bnode(struct kfree_rcu_cpu *krcp,
struct kvfree_rcu_bulk_data *bnode)
{
- // Check the limit.
- if (krcp->nr_bkv_objs >= rcu_min_cached_objs)
- return false;
-
- llist_add((struct llist_node *) bnode, &krcp->bkvcache);
- krcp->nr_bkv_objs++;
- return true;
+ if (atomic_test_inc_dec(&krcp->nr_bkv_objs, rcu_min_cached_objs, true)) {
+ llist_add((struct llist_node *) bnode, &krcp->bkvcache);
+ return true;
+ }
+ return false;
}
static int
drain_page_cache(struct kfree_rcu_cpu *krcp)
{
- unsigned long flags;
- struct llist_node *page_list, *pos, *n;
- int freed = 0;
+ struct kvfree_rcu_bulk_data *bnode;
+ int num_pages, i;
- raw_spin_lock_irqsave(&krcp->lock, flags);
- page_list = llist_del_all(&krcp->bkvcache);
- krcp->nr_bkv_objs = 0;
- raw_spin_unlock_irqrestore(&krcp->lock, flags);
+ num_pages = atomic_read(&krcp->nr_bkv_objs);
+
+ for (i = 0; i < num_pages; i++) {
+ bnode = get_cached_bnode(krcp);
+ if (!bnode)
+ break;
- llist_for_each_safe(pos, n, page_list) {
- free_page((unsigned long)pos);
- freed++;
+ free_page((unsigned long) bnode);
}
- return freed;
+ return i;
}
/*
@@ -3314,10 +3345,8 @@ static void kfree_rcu_work(struct work_struct *work)
}
rcu_lock_release(&rcu_callback_map);
- raw_spin_lock_irqsave(&krcp->lock, flags);
if (put_cached_bnode(krcp, bkvhead[i]))
bkvhead[i] = NULL;
- raw_spin_unlock_irqrestore(&krcp->lock, flags);
if (bkvhead[i])
free_page((unsigned long) bkvhead[i]);
@@ -3460,7 +3489,6 @@ static void fill_page_cache_func(struct work_struct *work)
struct kfree_rcu_cpu *krcp =
container_of(work, struct kfree_rcu_cpu,
page_cache_work.work);
- unsigned long flags;
bool pushed;
int i;
@@ -3469,10 +3497,7 @@ static void fill_page_cache_func(struct work_struct *work)
__get_free_page(GFP_KERNEL | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
if (bnode) {
- raw_spin_lock_irqsave(&krcp->lock, flags);
pushed = put_cached_bnode(krcp, bnode);
- raw_spin_unlock_irqrestore(&krcp->lock, flags);
-
if (!pushed) {
free_page((unsigned long) bnode);
break;
@@ -3647,17 +3672,13 @@ kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
{
int cpu;
unsigned long count = 0;
- unsigned long flags;
/* Snapshot count of all CPUs */
for_each_possible_cpu(cpu) {
struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
count += READ_ONCE(krcp->count);
-
- raw_spin_lock_irqsave(&krcp->lock, flags);
- count += krcp->nr_bkv_objs;
- raw_spin_unlock_irqrestore(&krcp->lock, flags);
+ count += atomic_read(&krcp->nr_bkv_objs);
}
WRITE_ONCE(backoff_page_cache_fill, true);
--
2.20.1
^ permalink raw reply related [flat|nested] 9+ messages in thread
* Re: [PATCH 1/2] kvfree_rcu: Release a page cache under memory pressure
2021-03-08 19:49 [PATCH 1/2] kvfree_rcu: Release a page cache under memory pressure Uladzislau Rezki (Sony)
@ 2021-03-08 23:37 ` kernel test robot
2021-03-08 23:37 ` kernel test robot
` (2 subsequent siblings)
3 siblings, 0 replies; 9+ messages in thread
From: kernel test robot @ 2021-03-08 23:37 UTC (permalink / raw)
To: Uladzislau Rezki (Sony), LKML, RCU, Paul E . McKenney
Cc: kbuild-all, clang-built-linux, Michal Hocko, Andrew Morton,
Linux Memory Management List, Daniel Axtens, Frederic Weisbecker,
Neeraj Upadhyay, Joel Fernandes
[-- Attachment #1: Type: text/plain, Size: 3576 bytes --]
Hi "Uladzislau,
Thank you for the patch! Perhaps something to improve:
[auto build test WARNING on rcu/dev]
[also build test WARNING on linux/master linus/master v5.12-rc2 next-20210305]
[cannot apply to hnaz-linux-mm/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Uladzislau-Rezki-Sony/kvfree_rcu-Release-a-page-cache-under-memory-pressure/20210309-035155
base: https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git dev
config: riscv-randconfig-r002-20210309 (attached as .config)
compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project 3a11a41795bec548e91621caaa4cc00fc31b2212)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# install riscv cross compiling tool for clang build
# apt-get install binutils-riscv64-linux-gnu
# https://github.com/0day-ci/linux/commit/d1c3a54d06b2b088b2a7b4e7f2a0a85bcbfbf53a
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Uladzislau-Rezki-Sony/kvfree_rcu-Release-a-page-cache-under-memory-pressure/20210309-035155
git checkout d1c3a54d06b2b088b2a7b4e7f2a0a85bcbfbf53a
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=riscv
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All warnings (new ones prefixed by >>):
>> kernel/rcu/tree.c:3518:7: warning: variable '__ret' is uninitialized when used here [-Wuninitialized]
if (xchg(&backoff_page_cache_fill, false)) {
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
arch/riscv/include/asm/cmpxchg.h:146:23: note: expanded from macro 'xchg'
(__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
arch/riscv/include/asm/cmpxchg.h:140:2: note: expanded from macro '__xchg'
__ret; \
^~~~~
kernel/rcu/tree.c:3518:7: note: variable '__ret' is declared here
arch/riscv/include/asm/cmpxchg.h:146:23: note: expanded from macro 'xchg'
(__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \
^
arch/riscv/include/asm/cmpxchg.h:121:2: note: expanded from macro '__xchg'
__typeof__(*(ptr)) __ret; \
^
1 warning generated.
vim +/__ret +3518 kernel/rcu/tree.c
3512
3513 static void
3514 run_page_cache_worker(struct kfree_rcu_cpu *krcp)
3515 {
3516 if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
3517 !atomic_xchg(&krcp->work_in_progress, 1)) {
> 3518 if (xchg(&backoff_page_cache_fill, false)) {
3519 queue_delayed_work(system_wq,
3520 &krcp->page_cache_work, PAGE_CACHE_FILL_DELAY);
3521 } else {
3522 hrtimer_init(&krcp->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3523 krcp->hrtimer.function = schedule_page_work_fn;
3524 hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);
3525 }
3526 }
3527 }
3528
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 31435 bytes --]
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH 1/2] kvfree_rcu: Release a page cache under memory pressure
@ 2021-03-08 23:37 ` kernel test robot
0 siblings, 0 replies; 9+ messages in thread
From: kernel test robot @ 2021-03-08 23:37 UTC (permalink / raw)
To: kbuild-all
[-- Attachment #1: Type: text/plain, Size: 3652 bytes --]
Hi "Uladzislau,
Thank you for the patch! Perhaps something to improve:
[auto build test WARNING on rcu/dev]
[also build test WARNING on linux/master linus/master v5.12-rc2 next-20210305]
[cannot apply to hnaz-linux-mm/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Uladzislau-Rezki-Sony/kvfree_rcu-Release-a-page-cache-under-memory-pressure/20210309-035155
base: https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git dev
config: riscv-randconfig-r002-20210309 (attached as .config)
compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project 3a11a41795bec548e91621caaa4cc00fc31b2212)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# install riscv cross compiling tool for clang build
# apt-get install binutils-riscv64-linux-gnu
# https://github.com/0day-ci/linux/commit/d1c3a54d06b2b088b2a7b4e7f2a0a85bcbfbf53a
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Uladzislau-Rezki-Sony/kvfree_rcu-Release-a-page-cache-under-memory-pressure/20210309-035155
git checkout d1c3a54d06b2b088b2a7b4e7f2a0a85bcbfbf53a
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=riscv
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All warnings (new ones prefixed by >>):
>> kernel/rcu/tree.c:3518:7: warning: variable '__ret' is uninitialized when used here [-Wuninitialized]
if (xchg(&backoff_page_cache_fill, false)) {
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
arch/riscv/include/asm/cmpxchg.h:146:23: note: expanded from macro 'xchg'
(__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
arch/riscv/include/asm/cmpxchg.h:140:2: note: expanded from macro '__xchg'
__ret; \
^~~~~
kernel/rcu/tree.c:3518:7: note: variable '__ret' is declared here
arch/riscv/include/asm/cmpxchg.h:146:23: note: expanded from macro 'xchg'
(__typeof__(*(ptr))) __xchg((ptr), _x_, sizeof(*(ptr))); \
^
arch/riscv/include/asm/cmpxchg.h:121:2: note: expanded from macro '__xchg'
__typeof__(*(ptr)) __ret; \
^
1 warning generated.
vim +/__ret +3518 kernel/rcu/tree.c
3512
3513 static void
3514 run_page_cache_worker(struct kfree_rcu_cpu *krcp)
3515 {
3516 if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
3517 !atomic_xchg(&krcp->work_in_progress, 1)) {
> 3518 if (xchg(&backoff_page_cache_fill, false)) {
3519 queue_delayed_work(system_wq,
3520 &krcp->page_cache_work, PAGE_CACHE_FILL_DELAY);
3521 } else {
3522 hrtimer_init(&krcp->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
3523 krcp->hrtimer.function = schedule_page_work_fn;
3524 hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);
3525 }
3526 }
3527 }
3528
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
[-- Attachment #2: config.gz --]
[-- Type: application/gzip, Size: 31435 bytes --]
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH 1/2] kvfree_rcu: Release a page cache under memory pressure
2021-03-08 19:49 [PATCH 1/2] kvfree_rcu: Release a page cache under memory pressure Uladzislau Rezki (Sony)
@ 2021-03-08 23:44 ` kernel test robot
2021-03-08 23:37 ` kernel test robot
` (2 subsequent siblings)
3 siblings, 0 replies; 9+ messages in thread
From: kernel test robot @ 2021-03-08 23:44 UTC (permalink / raw)
To: Uladzislau Rezki (Sony), LKML, RCU, Paul E . McKenney
Cc: kbuild-all, Michal Hocko, Andrew Morton,
Linux Memory Management List, Daniel Axtens, Frederic Weisbecker,
Neeraj Upadhyay, Joel Fernandes
[-- Attachment #1: Type: text/plain, Size: 1854 bytes --]
Hi "Uladzislau,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on rcu/dev]
[also build test ERROR on linux/master linus/master v5.12-rc2 next-20210305]
[cannot apply to hnaz-linux-mm/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Uladzislau-Rezki-Sony/kvfree_rcu-Release-a-page-cache-under-memory-pressure/20210309-035155
base: https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git dev
config: arc-defconfig (attached as .config)
compiler: arc-elf-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/d1c3a54d06b2b088b2a7b4e7f2a0a85bcbfbf53a
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Uladzislau-Rezki-Sony/kvfree_rcu-Release-a-page-cache-under-memory-pressure/20210309-035155
git checkout d1c3a54d06b2b088b2a7b4e7f2a0a85bcbfbf53a
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=arc
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All errors (new ones prefixed by >>):
arc-elf-ld: kernel/rcu/tree.o: in function `kvfree_call_rcu':
>> tree.c:(.text+0x291a): undefined reference to `__xchg_bad_pointer'
>> arc-elf-ld: tree.c:(.text+0x291a): undefined reference to `__xchg_bad_pointer'
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 9311 bytes --]
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH 1/2] kvfree_rcu: Release a page cache under memory pressure
@ 2021-03-08 23:44 ` kernel test robot
0 siblings, 0 replies; 9+ messages in thread
From: kernel test robot @ 2021-03-08 23:44 UTC (permalink / raw)
To: kbuild-all
[-- Attachment #1: Type: text/plain, Size: 1893 bytes --]
Hi "Uladzislau,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on rcu/dev]
[also build test ERROR on linux/master linus/master v5.12-rc2 next-20210305]
[cannot apply to hnaz-linux-mm/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Uladzislau-Rezki-Sony/kvfree_rcu-Release-a-page-cache-under-memory-pressure/20210309-035155
base: https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git dev
config: arc-defconfig (attached as .config)
compiler: arc-elf-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/d1c3a54d06b2b088b2a7b4e7f2a0a85bcbfbf53a
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Uladzislau-Rezki-Sony/kvfree_rcu-Release-a-page-cache-under-memory-pressure/20210309-035155
git checkout d1c3a54d06b2b088b2a7b4e7f2a0a85bcbfbf53a
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=arc
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All errors (new ones prefixed by >>):
arc-elf-ld: kernel/rcu/tree.o: in function `kvfree_call_rcu':
>> tree.c:(.text+0x291a): undefined reference to `__xchg_bad_pointer'
>> arc-elf-ld: tree.c:(.text+0x291a): undefined reference to `__xchg_bad_pointer'
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
[-- Attachment #2: config.gz --]
[-- Type: application/gzip, Size: 9311 bytes --]
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH 1/2] kvfree_rcu: Release a page cache under memory pressure
2021-03-08 19:49 [PATCH 1/2] kvfree_rcu: Release a page cache under memory pressure Uladzislau Rezki (Sony)
@ 2021-03-08 23:52 ` kernel test robot
2021-03-08 23:37 ` kernel test robot
` (2 subsequent siblings)
3 siblings, 0 replies; 9+ messages in thread
From: kernel test robot @ 2021-03-08 23:52 UTC (permalink / raw)
To: Uladzislau Rezki (Sony), LKML, RCU, Paul E . McKenney
Cc: kbuild-all, Michal Hocko, Andrew Morton,
Linux Memory Management List, Daniel Axtens, Frederic Weisbecker,
Neeraj Upadhyay, Joel Fernandes
[-- Attachment #1: Type: text/plain, Size: 1868 bytes --]
Hi "Uladzislau,
Thank you for the patch! Perhaps something to improve:
[auto build test WARNING on rcu/dev]
[also build test WARNING on linux/master linus/master v5.12-rc2 next-20210305]
[cannot apply to hnaz-linux-mm/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Uladzislau-Rezki-Sony/kvfree_rcu-Release-a-page-cache-under-memory-pressure/20210309-035155
base: https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git dev
config: h8300-randconfig-m031-20210308 (attached as .config)
compiler: h8300-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/d1c3a54d06b2b088b2a7b4e7f2a0a85bcbfbf53a
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Uladzislau-Rezki-Sony/kvfree_rcu-Release-a-page-cache-under-memory-pressure/20210309-035155
git checkout d1c3a54d06b2b088b2a7b4e7f2a0a85bcbfbf53a
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=h8300
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All warnings (new ones prefixed by >>):
/tmp/cctEAUUM.s: Assembler messages:
>> /tmp/cctEAUUM.s:306: Warning: mismatch between opcode size and operand size
/tmp/cctEAUUM.s:307: Warning: mismatch between opcode size and operand size
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 24513 bytes --]
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH 1/2] kvfree_rcu: Release a page cache under memory pressure
@ 2021-03-08 23:52 ` kernel test robot
0 siblings, 0 replies; 9+ messages in thread
From: kernel test robot @ 2021-03-08 23:52 UTC (permalink / raw)
To: kbuild-all
[-- Attachment #1: Type: text/plain, Size: 1907 bytes --]
Hi "Uladzislau,
Thank you for the patch! Perhaps something to improve:
[auto build test WARNING on rcu/dev]
[also build test WARNING on linux/master linus/master v5.12-rc2 next-20210305]
[cannot apply to hnaz-linux-mm/master]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Uladzislau-Rezki-Sony/kvfree_rcu-Release-a-page-cache-under-memory-pressure/20210309-035155
base: https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git dev
config: h8300-randconfig-m031-20210308 (attached as .config)
compiler: h8300-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/d1c3a54d06b2b088b2a7b4e7f2a0a85bcbfbf53a
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Uladzislau-Rezki-Sony/kvfree_rcu-Release-a-page-cache-under-memory-pressure/20210309-035155
git checkout d1c3a54d06b2b088b2a7b4e7f2a0a85bcbfbf53a
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=h8300
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All warnings (new ones prefixed by >>):
/tmp/cctEAUUM.s: Assembler messages:
>> /tmp/cctEAUUM.s:306: Warning: mismatch between opcode size and operand size
/tmp/cctEAUUM.s:307: Warning: mismatch between opcode size and operand size
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
[-- Attachment #2: config.gz --]
[-- Type: application/gzip, Size: 24513 bytes --]
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH 2/2] kvfree_rcu: convert a page cache to lock-free variant
[not found] ` <CA+KHdyX-h2KHEEvZmbeLVdjBzW7o37_GvQrGj55-d4i-+HLboQ@mail.gmail.com>
@ 2021-03-12 14:49 ` Paul E. McKenney
0 siblings, 0 replies; 9+ messages in thread
From: Paul E. McKenney @ 2021-03-12 14:49 UTC (permalink / raw)
To: Uladzislau Rezki
Cc: Michal Hocko, Andrew Morton, Daniel Axtens, Frederic Weisbecker,
Neeraj Upadhyay, Joel Fernandes, Peter Zijlstra, Thomas Gleixner,
Theodore Y . Ts'o, Sebastian Andrzej Siewior,
Oleksiy Avramchenko, RCU, LKML
On Fri, Mar 12, 2021 at 01:38:14PM +0100, Uladzislau Rezki wrote:
> Hello, Paul.
>
> Please do not consider this patch. It is buggy :)
Consider it formally withdrawn, and thank you for letting me know!
Thanx, Paul
> --
> Vlad Rezki
>
> On Mon, Mar 8, 2021 at 8:50 PM Uladzislau Rezki (Sony) <urezki@gmail.com>
> wrote:
>
> > Implement an access to the page cache as lock-free variant. This
> > is done because there are extra places where an access is required,
> > therefore making it lock-less will remove any lock contention.
> >
> > For example we have a shrinker path as well as a reclaim kthread.
> > In both cases a current CPU can access to a remote per-cpu page
> > cache that would require taking a lock to protect it.
> >
> > An "rcuscale" performance test suite can detect it and shows some
> > slight improvements:
> >
> > ../kvm.sh --memory 16G --torture rcuscale --allcpus --duration 10 \
> > --kconfig CONFIG_NR_CPUS=64 --bootargs "rcuscale.kfree_rcu_test=1 \
> > rcuscale.kfree_nthreads=16 rcuscale.holdoff=20 rcuscale.kfree_loops=10000 \
> > rcuscale.kfree_rcu_test_double=1 torture.disable_onoff_at_boot"
> > --trust-make
> >
> > 100 iterations, checking total time taken by all kfree'ers in ns.:
> >
> > default: AVG: 10968415107.5 MIN: 10668412500 MAX: 11312145160
> > patch: AVG: 10787596486.1 MIN: 10397559880 MAX: 11214901050
> >
> > Signed-off-by: Uladzislau Rezki (Sony) <urezki@gmail.com>
> > ---
> > kernel/rcu/tree.c | 91 +++++++++++++++++++++++++++++------------------
> > 1 file changed, 56 insertions(+), 35 deletions(-)
> >
> > diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> > index 9c8cfb01e9a6..4f04664d5ac0 100644
> > --- a/kernel/rcu/tree.c
> > +++ b/kernel/rcu/tree.c
> > @@ -3167,8 +3167,9 @@ struct kfree_rcu_cpu {
> > atomic_t work_in_progress;
> > struct hrtimer hrtimer;
> >
> > + // lock-free cache.
> > struct llist_head bkvcache;
> > - int nr_bkv_objs;
> > + atomic_t nr_bkv_objs;
> > };
> >
> > static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc) = {
> > @@ -3215,49 +3216,79 @@ krc_this_cpu_unlock(struct kfree_rcu_cpu *krcp,
> > unsigned long flags)
> > raw_spin_unlock_irqrestore(&krcp->lock, flags);
> > }
> >
> > +/*
> > + * Increment 'v', if 'v' is below 'thresh'. Returns true if we
> > + * succeeded, false if 'v' + 1 would be bigger than 'thresh'.
> > + *
> > + * Decrement 'v' if 'v' is upper 'thresh'. Returns true if we
> > + * succeeded, false if 'v' - 1 would be smaller than 'thresh'.
> > + */
> > +static inline bool
> > +atomic_test_inc_dec(atomic_t *v, unsigned int thresh, bool inc)
> > +{
> > + unsigned int cur = atomic_read(v);
> > + unsigned int old;
> > +
> > + for (;;) {
> > + if (inc) {
> > + if (cur >= thresh)
> > + return false;
> > + } else {
> > + if (cur <= thresh)
> > + return false;
> > + }
> > +
> > + old = atomic_cmpxchg(v, cur, inc ? (cur + 1):(cur - 1));
> > + if (old == cur)
> > + break;
> > +
> > + cur = old;
> > + }
> > +
> > + return true;
> > +}
> > +
> > static inline struct kvfree_rcu_bulk_data *
> > get_cached_bnode(struct kfree_rcu_cpu *krcp)
> > {
> > - if (!krcp->nr_bkv_objs)
> > - return NULL;
> > + struct kvfree_rcu_bulk_data *bnode = NULL;
> >
> > - krcp->nr_bkv_objs--;
> > - return (struct kvfree_rcu_bulk_data *)
> > - llist_del_first(&krcp->bkvcache);
> > + if (atomic_test_inc_dec(&krcp->nr_bkv_objs, 0, false))
> > + bnode = (struct kvfree_rcu_bulk_data *)
> > + llist_del_first(&krcp->bkvcache);
> > +
> > + return bnode;
> > }
> >
> > static inline bool
> > put_cached_bnode(struct kfree_rcu_cpu *krcp,
> > struct kvfree_rcu_bulk_data *bnode)
> > {
> > - // Check the limit.
> > - if (krcp->nr_bkv_objs >= rcu_min_cached_objs)
> > - return false;
> > -
> > - llist_add((struct llist_node *) bnode, &krcp->bkvcache);
> > - krcp->nr_bkv_objs++;
> > - return true;
> > + if (atomic_test_inc_dec(&krcp->nr_bkv_objs, rcu_min_cached_objs,
> > true)) {
> > + llist_add((struct llist_node *) bnode, &krcp->bkvcache);
> > + return true;
> > + }
> >
> > + return false;
> > }
> >
> > static int
> > drain_page_cache(struct kfree_rcu_cpu *krcp)
> > {
> > - unsigned long flags;
> > - struct llist_node *page_list, *pos, *n;
> > - int freed = 0;
> > + struct kvfree_rcu_bulk_data *bnode;
> > + int num_pages, i;
> >
> > - raw_spin_lock_irqsave(&krcp->lock, flags);
> > - page_list = llist_del_all(&krcp->bkvcache);
> > - krcp->nr_bkv_objs = 0;
> > - raw_spin_unlock_irqrestore(&krcp->lock, flags);
> > + num_pages = atomic_read(&krcp->nr_bkv_objs);
> > +
> > + for (i = 0; i < num_pages; i++) {
> > + bnode = get_cached_bnode(krcp);
> > + if (!bnode)
> > + break;
> >
> > - llist_for_each_safe(pos, n, page_list) {
> > - free_page((unsigned long)pos);
> > - freed++;
> > + free_page((unsigned long) bnode);
> > }
> >
> > - return freed;
> > + return i;
> > }
> >
> > /*
> > @@ -3314,10 +3345,8 @@ static void kfree_rcu_work(struct work_struct *work)
> > }
> > rcu_lock_release(&rcu_callback_map);
> >
> > - raw_spin_lock_irqsave(&krcp->lock, flags);
> > if (put_cached_bnode(krcp, bkvhead[i]))
> > bkvhead[i] = NULL;
> > - raw_spin_unlock_irqrestore(&krcp->lock, flags);
> >
> > if (bkvhead[i])
> > free_page((unsigned long) bkvhead[i]);
> > @@ -3460,7 +3489,6 @@ static void fill_page_cache_func(struct work_struct
> > *work)
> > struct kfree_rcu_cpu *krcp =
> > container_of(work, struct kfree_rcu_cpu,
> > page_cache_work.work);
> > - unsigned long flags;
> > bool pushed;
> > int i;
> >
> > @@ -3469,10 +3497,7 @@ static void fill_page_cache_func(struct work_struct
> > *work)
> > __get_free_page(GFP_KERNEL | __GFP_NORETRY |
> > __GFP_NOMEMALLOC | __GFP_NOWARN);
> >
> > if (bnode) {
> > - raw_spin_lock_irqsave(&krcp->lock, flags);
> > pushed = put_cached_bnode(krcp, bnode);
> > - raw_spin_unlock_irqrestore(&krcp->lock, flags);
> > -
> > if (!pushed) {
> > free_page((unsigned long) bnode);
> > break;
> > @@ -3647,17 +3672,13 @@ kfree_rcu_shrink_count(struct shrinker *shrink,
> > struct shrink_control *sc)
> > {
> > int cpu;
> > unsigned long count = 0;
> > - unsigned long flags;
> >
> > /* Snapshot count of all CPUs */
> > for_each_possible_cpu(cpu) {
> > struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
> >
> > count += READ_ONCE(krcp->count);
> > -
> > - raw_spin_lock_irqsave(&krcp->lock, flags);
> > - count += krcp->nr_bkv_objs;
> > - raw_spin_unlock_irqrestore(&krcp->lock, flags);
> > + count += atomic_read(&krcp->nr_bkv_objs);
> > }
> >
> > WRITE_ONCE(backoff_page_cache_fill, true);
> > --
> > 2.20.1
> >
> >
>
> --
> Uladzislau Rezki
^ permalink raw reply [flat|nested] 9+ messages in thread
end of thread, other threads:[~2021-03-12 14:50 UTC | newest]
Thread overview: 9+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-03-08 19:49 [PATCH 1/2] kvfree_rcu: Release a page cache under memory pressure Uladzislau Rezki (Sony)
2021-03-08 19:49 ` [PATCH 2/2] kvfree_rcu: convert a page cache to lock-free variant Uladzislau Rezki (Sony)
[not found] ` <CA+KHdyX-h2KHEEvZmbeLVdjBzW7o37_GvQrGj55-d4i-+HLboQ@mail.gmail.com>
2021-03-12 14:49 ` Paul E. McKenney
2021-03-08 23:37 ` [PATCH 1/2] kvfree_rcu: Release a page cache under memory pressure kernel test robot
2021-03-08 23:37 ` kernel test robot
2021-03-08 23:44 ` kernel test robot
2021-03-08 23:44 ` kernel test robot
2021-03-08 23:52 ` kernel test robot
2021-03-08 23:52 ` kernel test robot
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.