rcu.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
* [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
@ 2021-01-21  6:49 qiang.zhang
  2021-01-21 18:56 ` Paul E. McKenney
  0 siblings, 1 reply; 10+ messages in thread
From: qiang.zhang @ 2021-01-21  6:49 UTC (permalink / raw)
  To: paulmck; +Cc: rcu, linux-kernel

From: Zqiang <qiang.zhang@windriver.com>

If CPUs go offline, the corresponding krcp's page cache can
not be use util the CPU come back online, or maybe the CPU
will never go online again, this commit therefore free krcp's
page cache when CPUs go offline.

Signed-off-by: Zqiang <qiang.zhang@windriver.com>
---
 kernel/rcu/tree.c | 47 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 46 insertions(+), 1 deletion(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index e04e336bee42..2eaf6f287483 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -158,6 +158,9 @@ static void sync_sched_exp_online_cleanup(int cpu);
 static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);
 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp);
 
+static void krc_offline(unsigned int cpu, bool off);
+static void free_krc_page_cache(int cpu);
+
 /* rcuc/rcub kthread realtime priority */
 static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
 module_param(kthread_prio, int, 0444);
@@ -2457,6 +2460,9 @@ int rcutree_dead_cpu(unsigned int cpu)
 
 	// Stop-machine done, so allow nohz_full to disable tick.
 	tick_dep_clear(TICK_DEP_BIT_RCU);
+
+	krc_offline(cpu, true);
+	free_krc_page_cache(cpu);
 	return 0;
 }
 
@@ -3169,6 +3175,7 @@ struct kfree_rcu_cpu {
 
 	struct llist_head bkvcache;
 	int nr_bkv_objs;
+	bool offline;
 };
 
 static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc) = {
@@ -3220,6 +3227,8 @@ static inline bool
 put_cached_bnode(struct kfree_rcu_cpu *krcp,
 	struct kvfree_rcu_bulk_data *bnode)
 {
+	if (krcp->offline)
+		return false;
 	// Check the limit.
 	if (krcp->nr_bkv_objs >= rcu_min_cached_objs)
 		return false;
@@ -3230,6 +3239,39 @@ put_cached_bnode(struct kfree_rcu_cpu *krcp,
 
 }
 
+static void krc_offline(unsigned int cpu, bool off)
+{
+	unsigned long flags;
+	struct kfree_rcu_cpu *krcp;
+
+	krcp = per_cpu_ptr(&krc, cpu);
+	raw_spin_lock_irqsave(&krcp->lock, flags);
+	if (off)
+		krcp->offline = true;
+	else
+		krcp->offline = false;
+	raw_spin_unlock_irqrestore(&krcp->lock, flags);
+}
+
+static void free_krc_page_cache(int cpu)
+{
+	unsigned long flags;
+	struct kfree_rcu_cpu *krcp;
+	int i;
+	struct kvfree_rcu_bulk_data *bnode;
+
+	krcp = per_cpu_ptr(&krc, cpu);
+
+	for (i = 0; i < rcu_min_cached_objs; i++) {
+		raw_spin_lock_irqsave(&krcp->lock, flags);
+		bnode = get_cached_bnode(krcp);
+		raw_spin_unlock_irqrestore(&krcp->lock, flags);
+		if (!bnode)
+			break;
+		free_page((unsigned long)bnode);
+	}
+}
+
 /*
  * This function is invoked in workqueue context after a grace period.
  * It frees all the objects queued on ->bhead_free or ->head_free.
@@ -3549,7 +3591,8 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
 	kasan_record_aux_stack(ptr);
 	success = kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr);
 	if (!success) {
-		run_page_cache_worker(krcp);
+		if (!krcp->offline)
+			run_page_cache_worker(krcp);
 
 		if (head == NULL)
 			// Inline if kvfree_rcu(one_arg) call.
@@ -4086,6 +4129,7 @@ int rcutree_prepare_cpu(unsigned int cpu)
 	rcu_spawn_cpu_nocb_kthread(cpu);
 	WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus + 1);
 
+	krc_offline(cpu, false);
 	return 0;
 }
 
@@ -4591,6 +4635,7 @@ static void __init kfree_rcu_batch_init(void)
 		INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
 		INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
 		krcp->initialized = true;
+		krcp->offline = true;
 	}
 	if (register_shrinker(&kfree_rcu_shrinker))
 		pr_err("Failed to register kfree_rcu() shrinker!\n");
-- 
2.29.2


^ permalink raw reply related	[flat|nested] 10+ messages in thread

* Re: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
  2021-01-21  6:49 [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline qiang.zhang
@ 2021-01-21 18:56 ` Paul E. McKenney
  2021-01-21 20:26   ` Uladzislau Rezki
  0 siblings, 1 reply; 10+ messages in thread
From: Paul E. McKenney @ 2021-01-21 18:56 UTC (permalink / raw)
  To: qiang.zhang; +Cc: rcu, linux-kernel, urezki

On Thu, Jan 21, 2021 at 02:49:49PM +0800, qiang.zhang@windriver.com wrote:
> From: Zqiang <qiang.zhang@windriver.com>
> 
> If CPUs go offline, the corresponding krcp's page cache can
> not be use util the CPU come back online, or maybe the CPU
> will never go online again, this commit therefore free krcp's
> page cache when CPUs go offline.
> 
> Signed-off-by: Zqiang <qiang.zhang@windriver.com>

Adding Uladzislau on CC for his thoughts.

							Thanx, Paul

> ---
>  kernel/rcu/tree.c | 47 ++++++++++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 46 insertions(+), 1 deletion(-)
> 
> diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> index e04e336bee42..2eaf6f287483 100644
> --- a/kernel/rcu/tree.c
> +++ b/kernel/rcu/tree.c
> @@ -158,6 +158,9 @@ static void sync_sched_exp_online_cleanup(int cpu);
>  static void check_cb_ovld_locked(struct rcu_data *rdp, struct rcu_node *rnp);
>  static bool rcu_rdp_is_offloaded(struct rcu_data *rdp);
>  
> +static void krc_offline(unsigned int cpu, bool off);
> +static void free_krc_page_cache(int cpu);
> +
>  /* rcuc/rcub kthread realtime priority */
>  static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
>  module_param(kthread_prio, int, 0444);
> @@ -2457,6 +2460,9 @@ int rcutree_dead_cpu(unsigned int cpu)
>  
>  	// Stop-machine done, so allow nohz_full to disable tick.
>  	tick_dep_clear(TICK_DEP_BIT_RCU);
> +
> +	krc_offline(cpu, true);
> +	free_krc_page_cache(cpu);
>  	return 0;
>  }
>  
> @@ -3169,6 +3175,7 @@ struct kfree_rcu_cpu {
>  
>  	struct llist_head bkvcache;
>  	int nr_bkv_objs;
> +	bool offline;
>  };
>  
>  static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc) = {
> @@ -3220,6 +3227,8 @@ static inline bool
>  put_cached_bnode(struct kfree_rcu_cpu *krcp,
>  	struct kvfree_rcu_bulk_data *bnode)
>  {
> +	if (krcp->offline)
> +		return false;
>  	// Check the limit.
>  	if (krcp->nr_bkv_objs >= rcu_min_cached_objs)
>  		return false;
> @@ -3230,6 +3239,39 @@ put_cached_bnode(struct kfree_rcu_cpu *krcp,
>  
>  }
>  
> +static void krc_offline(unsigned int cpu, bool off)
> +{
> +	unsigned long flags;
> +	struct kfree_rcu_cpu *krcp;
> +
> +	krcp = per_cpu_ptr(&krc, cpu);
> +	raw_spin_lock_irqsave(&krcp->lock, flags);
> +	if (off)
> +		krcp->offline = true;
> +	else
> +		krcp->offline = false;
> +	raw_spin_unlock_irqrestore(&krcp->lock, flags);
> +}
> +
> +static void free_krc_page_cache(int cpu)
> +{
> +	unsigned long flags;
> +	struct kfree_rcu_cpu *krcp;
> +	int i;
> +	struct kvfree_rcu_bulk_data *bnode;
> +
> +	krcp = per_cpu_ptr(&krc, cpu);
> +
> +	for (i = 0; i < rcu_min_cached_objs; i++) {
> +		raw_spin_lock_irqsave(&krcp->lock, flags);
> +		bnode = get_cached_bnode(krcp);
> +		raw_spin_unlock_irqrestore(&krcp->lock, flags);
> +		if (!bnode)
> +			break;
> +		free_page((unsigned long)bnode);
> +	}
> +}
> +
>  /*
>   * This function is invoked in workqueue context after a grace period.
>   * It frees all the objects queued on ->bhead_free or ->head_free.
> @@ -3549,7 +3591,8 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
>  	kasan_record_aux_stack(ptr);
>  	success = kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr);
>  	if (!success) {
> -		run_page_cache_worker(krcp);
> +		if (!krcp->offline)
> +			run_page_cache_worker(krcp);
>  
>  		if (head == NULL)
>  			// Inline if kvfree_rcu(one_arg) call.
> @@ -4086,6 +4129,7 @@ int rcutree_prepare_cpu(unsigned int cpu)
>  	rcu_spawn_cpu_nocb_kthread(cpu);
>  	WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus + 1);
>  
> +	krc_offline(cpu, false);
>  	return 0;
>  }
>  
> @@ -4591,6 +4635,7 @@ static void __init kfree_rcu_batch_init(void)
>  		INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
>  		INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
>  		krcp->initialized = true;
> +		krcp->offline = true;
>  	}
>  	if (register_shrinker(&kfree_rcu_shrinker))
>  		pr_err("Failed to register kfree_rcu() shrinker!\n");
> -- 
> 2.29.2
> 

^ permalink raw reply	[flat|nested] 10+ messages in thread

* Re: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
  2021-01-21 18:56 ` Paul E. McKenney
@ 2021-01-21 20:26   ` Uladzislau Rezki
  2021-01-22  1:44     ` 回复: " Zhang, Qiang
  0 siblings, 1 reply; 10+ messages in thread
From: Uladzislau Rezki @ 2021-01-21 20:26 UTC (permalink / raw)
  To: qiang.zhang; +Cc: Paul E. McKenney, rcu, linux-kernel, urezki

Hello, Qiang,

> On Thu, Jan 21, 2021 at 02:49:49PM +0800, qiang.zhang@windriver.com wrote:
> > From: Zqiang <qiang.zhang@windriver.com>
> > 
> > If CPUs go offline, the corresponding krcp's page cache can
> > not be use util the CPU come back online, or maybe the CPU
> > will never go online again, this commit therefore free krcp's
> > page cache when CPUs go offline.
> > 
> > Signed-off-by: Zqiang <qiang.zhang@windriver.com>
> 
Do you consider it as an issue? We have 5 pages per CPU, that is 20480 bytes. 

--
Vlad Rezki

^ permalink raw reply	[flat|nested] 10+ messages in thread

* 回复: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
  2021-01-21 20:26   ` Uladzislau Rezki
@ 2021-01-22  1:44     ` Zhang, Qiang
  2021-01-22 14:31       ` Uladzislau Rezki
  0 siblings, 1 reply; 10+ messages in thread
From: Zhang, Qiang @ 2021-01-22  1:44 UTC (permalink / raw)
  To: Uladzislau Rezki; +Cc: Paul E. McKenney, rcu, linux-kernel



________________________________________
发件人: Uladzislau Rezki <urezki@gmail.com>
发送时间: 2021年1月22日 4:26
收件人: Zhang, Qiang
抄送: Paul E. McKenney; rcu@vger.kernel.org; linux-kernel@vger.kernel.org; urezki@gmail.com
主题: Re: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
>Hello, Qiang,

> On Thu, Jan 21, 2021 at 02:49:49PM +0800, qiang.zhang@windriver.com wrote:
> > From: Zqiang <qiang.zhang@windriver.com>
> >
> > If CPUs go offline, the corresponding krcp's page cache can
> > not be use util the CPU come back online, or maybe the CPU
> > will never go online again, this commit therefore free krcp's
> > page cache when CPUs go offline.
> >
> > Signed-off-by: Zqiang <qiang.zhang@windriver.com>
>
>Do you consider it as an issue? We have 5 pages per CPU, that is 20480 bytes.
>

Hello Rezki 

In a multi CPUs system, more than one CPUs may be offline, there are more than 5 pages,  and these offline CPUs may never go online again  or  in the process of CPUs online, there are errors, which lead to the failure of online, these scenarios will lead to the per-cpu krc page cache will never be released.

Thanks
Qiang


>--
>Vlad Rezki

^ permalink raw reply	[flat|nested] 10+ messages in thread

* Re: 回复: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
  2021-01-22  1:44     ` 回复: " Zhang, Qiang
@ 2021-01-22 14:31       ` Uladzislau Rezki
  2021-01-24  2:21         ` 回复: " Zhang, Qiang
  0 siblings, 1 reply; 10+ messages in thread
From: Uladzislau Rezki @ 2021-01-22 14:31 UTC (permalink / raw)
  To: Zhang, Qiang; +Cc: Uladzislau Rezki, Paul E. McKenney, rcu, linux-kernel

On Fri, Jan 22, 2021 at 01:44:36AM +0000, Zhang, Qiang wrote:
> 
> 
> ________________________________________
> 发件人: Uladzislau Rezki <urezki@gmail.com>
> 发送时间: 2021年1月22日 4:26
> 收件人: Zhang, Qiang
> 抄送: Paul E. McKenney; rcu@vger.kernel.org; linux-kernel@vger.kernel.org; urezki@gmail.com
> 主题: Re: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
> >Hello, Qiang,
> 
> > On Thu, Jan 21, 2021 at 02:49:49PM +0800, qiang.zhang@windriver.com wrote:
> > > From: Zqiang <qiang.zhang@windriver.com>
> > >
> > > If CPUs go offline, the corresponding krcp's page cache can
> > > not be use util the CPU come back online, or maybe the CPU
> > > will never go online again, this commit therefore free krcp's
> > > page cache when CPUs go offline.
> > >
> > > Signed-off-by: Zqiang <qiang.zhang@windriver.com>
> >
> >Do you consider it as an issue? We have 5 pages per CPU, that is 20480 bytes.
> >
> 
> Hello Rezki 
> 
> In a multi CPUs system, more than one CPUs may be offline, there are more than 5 pages,  and these offline CPUs may never go online again  or  in the process of CPUs online, there are errors, which lead to the failure of online, these scenarios will lead to the per-cpu krc page cache will never be released.
> 
Thanks for your answer. I was thinking more about if you knew some platforms
which suffer from such extra page usage when CPU goes offline. Any issues
your platforms or devices run into because of that.

So i understand that if CPU goes offline the 5 pages associated with it are
unused until it goes online back.

--
Vlad Rezki

^ permalink raw reply	[flat|nested] 10+ messages in thread

* 回复: 回复: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
  2021-01-22 14:31       ` Uladzislau Rezki
@ 2021-01-24  2:21         ` Zhang, Qiang
  2021-01-24 17:42           ` Uladzislau Rezki
  2021-01-26 14:07           ` Uladzislau Rezki
  0 siblings, 2 replies; 10+ messages in thread
From: Zhang, Qiang @ 2021-01-24  2:21 UTC (permalink / raw)
  To: Uladzislau Rezki; +Cc: Paul E. McKenney, rcu, linux-kernel



________________________________________
发件人: Uladzislau Rezki <urezki@gmail.com>
发送时间: 2021年1月22日 22:31
收件人: Zhang, Qiang
抄送: Uladzislau Rezki; Paul E. McKenney; rcu@vger.kernel.org; linux-kernel@vger.kernel.org
主题: Re: 回复: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline

On Fri, Jan 22, 2021 at 01:44:36AM +0000, Zhang, Qiang wrote:
>
>
> ________________________________________
> 发件人: Uladzislau Rezki <urezki@gmail.com>
> 发送时间: 2021年1月22日 4:26
> 收件人: Zhang, Qiang
> 抄送: Paul E. McKenney; rcu@vger.kernel.org; linux-kernel@vger.kernel.org; urezki@gmail.com
> 主题: Re: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
> >Hello, Qiang,
>
> > On Thu, Jan 21, 2021 at 02:49:49PM +0800, qiang.zhang@windriver.com wrote:
> > > From: Zqiang <qiang.zhang@windriver.com>
> > >
> > > If CPUs go offline, the corresponding krcp's page cache can
> > > not be use util the CPU come back online, or maybe the CPU
> > > will never go online again, this commit therefore free krcp's
> > > page cache when CPUs go offline.
> > >
> > > Signed-off-by: Zqiang <qiang.zhang@windriver.com>
> >
> >Do you consider it as an issue? We have 5 pages per CPU, that is 20480 bytes.
> >
>
> Hello Rezki
>
> In a multi CPUs system, more than one CPUs may be offline, there are more than 5 pages,  and these offline CPUs may never go online again  or  in the process of CPUs online, there are errors, which lead to the failure of online, these scenarios will lead to the per-cpu krc page cache will never be released.
>
>Thanks for your answer. I was thinking more about if you knew some >platforms
>which suffer from such extra page usage when CPU goes offline. Any >issues
>your platforms or devices run into because of that.
>
>So i understand that if CPU goes offline the 5 pages associated with it >are
>unused until it goes online back.

 I agree with you, But I still want to talk about what I think

 My understanding is that when the CPU is offline,  the pages is not 
 accessible,  beacuse we don't know when this CPU will 
 go online again, so we best to return these page to the buddy system,
 when the CPU goes online again, we can allocate page from the buddy 
 system to fill krcp's page cache.  maybe you may think that this memory 
 is small and don't need to. 
 
 Thanks 
 Qiang

>
>--
>Vlad Rezki

^ permalink raw reply	[flat|nested] 10+ messages in thread

* Re: 回复: 回复: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
  2021-01-24  2:21         ` 回复: " Zhang, Qiang
@ 2021-01-24 17:42           ` Uladzislau Rezki
  2021-01-26 14:07           ` Uladzislau Rezki
  1 sibling, 0 replies; 10+ messages in thread
From: Uladzislau Rezki @ 2021-01-24 17:42 UTC (permalink / raw)
  To: Zhang, Qiang; +Cc: Uladzislau Rezki, Paul E. McKenney, rcu, linux-kernel

On Sun, Jan 24, 2021 at 02:21:07AM +0000, Zhang, Qiang wrote:
> 
> 
> ________________________________________
> 发件人: Uladzislau Rezki <urezki@gmail.com>
> 发送时间: 2021年1月22日 22:31
> 收件人: Zhang, Qiang
> 抄送: Uladzislau Rezki; Paul E. McKenney; rcu@vger.kernel.org; linux-kernel@vger.kernel.org
> 主题: Re: 回复: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
> 
> On Fri, Jan 22, 2021 at 01:44:36AM +0000, Zhang, Qiang wrote:
> >
> >
> > ________________________________________
> > 发件人: Uladzislau Rezki <urezki@gmail.com>
> > 发送时间: 2021年1月22日 4:26
> > 收件人: Zhang, Qiang
> > 抄送: Paul E. McKenney; rcu@vger.kernel.org; linux-kernel@vger.kernel.org; urezki@gmail.com
> > 主题: Re: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
> > >Hello, Qiang,
> >
> > > On Thu, Jan 21, 2021 at 02:49:49PM +0800, qiang.zhang@windriver.com wrote:
> > > > From: Zqiang <qiang.zhang@windriver.com>
> > > >
> > > > If CPUs go offline, the corresponding krcp's page cache can
> > > > not be use util the CPU come back online, or maybe the CPU
> > > > will never go online again, this commit therefore free krcp's
> > > > page cache when CPUs go offline.
> > > >
> > > > Signed-off-by: Zqiang <qiang.zhang@windriver.com>
> > >
> > >Do you consider it as an issue? We have 5 pages per CPU, that is 20480 bytes.
> > >
> >
> > Hello Rezki
> >
> > In a multi CPUs system, more than one CPUs may be offline, there are more than 5 pages,  and these offline CPUs may never go online again  or  in the process of CPUs online, there are errors, which lead to the failure of online, these scenarios will lead to the per-cpu krc page cache will never be released.
> >
> >Thanks for your answer. I was thinking more about if you knew some >platforms
> >which suffer from such extra page usage when CPU goes offline. Any >issues
> >your platforms or devices run into because of that.
> >
> >So i understand that if CPU goes offline the 5 pages associated with it >are
> >unused until it goes online back.
> 
>  I agree with you, But I still want to talk about what I think
> 
>  My understanding is that when the CPU is offline,  the pages is not 
>  accessible,  beacuse we don't know when this CPU will 
>  go online again, so we best to return these page to the buddy system,
>  when the CPU goes online again, we can allocate page from the buddy 
>  system to fill krcp's page cache.  maybe you may think that this memory 
>  is small and don't need to. 
>  
Exactly. I think that 5 pages are not that important(small amount of memory),
so it is not worth to build the logic around it in order to return them back
to the page allocator when CPU goes offline, even forever. This is how i see it.

If you are aware about some specific systems where it is critical, please
let's as know.

--
Vlad Rezki

^ permalink raw reply	[flat|nested] 10+ messages in thread

* Re: 回复: 回复: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
  2021-01-24  2:21         ` 回复: " Zhang, Qiang
  2021-01-24 17:42           ` Uladzislau Rezki
@ 2021-01-26 14:07           ` Uladzislau Rezki
  2021-01-27  9:00             ` 回复: " Zhang, Qiang
  1 sibling, 1 reply; 10+ messages in thread
From: Uladzislau Rezki @ 2021-01-26 14:07 UTC (permalink / raw)
  To: Zhang, Qiang; +Cc: Uladzislau Rezki, Paul E. McKenney, rcu, linux-kernel

> ________________________________________
> 发件人: Uladzislau Rezki <urezki@gmail.com>
> 发送时间: 2021年1月22日 22:31
> 收件人: Zhang, Qiang
> 抄送: Uladzislau Rezki; Paul E. McKenney; rcu@vger.kernel.org; linux-kernel@vger.kernel.org
> 主题: Re: 回复: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
> 
> On Fri, Jan 22, 2021 at 01:44:36AM +0000, Zhang, Qiang wrote:
> >
> >
> > ________________________________________
> > 发件人: Uladzislau Rezki <urezki@gmail.com>
> > 发送时间: 2021年1月22日 4:26
> > 收件人: Zhang, Qiang
> > 抄送: Paul E. McKenney; rcu@vger.kernel.org; linux-kernel@vger.kernel.org; urezki@gmail.com
> > 主题: Re: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
> > >Hello, Qiang,
> >
> > > On Thu, Jan 21, 2021 at 02:49:49PM +0800, qiang.zhang@windriver.com wrote:
> > > > From: Zqiang <qiang.zhang@windriver.com>
> > > >
> > > > If CPUs go offline, the corresponding krcp's page cache can
> > > > not be use util the CPU come back online, or maybe the CPU
> > > > will never go online again, this commit therefore free krcp's
> > > > page cache when CPUs go offline.
> > > >
> > > > Signed-off-by: Zqiang <qiang.zhang@windriver.com>
> > >
> > >Do you consider it as an issue? We have 5 pages per CPU, that is 20480 bytes.
> > >
> >
> > Hello Rezki
> >
> > In a multi CPUs system, more than one CPUs may be offline, there are more than 5 pages,  and these offline CPUs may never go online again  or  in the process of CPUs online, there are errors, which lead to the failure of online, these scenarios will lead to the per-cpu krc page cache will never be released.
> >
> >Thanks for your answer. I was thinking more about if you knew some >platforms
> >which suffer from such extra page usage when CPU goes offline. Any >issues
> >your platforms or devices run into because of that.
> >
> >So i understand that if CPU goes offline the 5 pages associated with it >are
> >unused until it goes online back.
> 
>  I agree with you, But I still want to talk about what I think
> 
>  My understanding is that when the CPU is offline,  the pages is not 
>  accessible,  beacuse we don't know when this CPU will 
>  go online again, so we best to return these page to the buddy system,
>  when the CPU goes online again, we can allocate page from the buddy 
>  system to fill krcp's page cache.  maybe you may think that this memory 
>  is small and don't need to. 
>  
BTW, we can release the caches via shrinker path instead, what is more makes
sense to me. We already have a callback, that frees pages when a page allocator
asks for it. I think in that case it would be fair to return it to the buddy
system. It happens under low memory condition or can be done manually to flush
system caches:

echo 3 > /proc/sys/vm/drop_caches

What do you think?

--
Vlad Rezki

^ permalink raw reply	[flat|nested] 10+ messages in thread

* 回复: 回复: 回复: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
  2021-01-26 14:07           ` Uladzislau Rezki
@ 2021-01-27  9:00             ` Zhang, Qiang
  2021-01-27 12:47               ` Uladzislau Rezki
  0 siblings, 1 reply; 10+ messages in thread
From: Zhang, Qiang @ 2021-01-27  9:00 UTC (permalink / raw)
  To: Uladzislau Rezki; +Cc: Paul E. McKenney, rcu, linux-kernel



________________________________________
发件人: Uladzislau Rezki <urezki@gmail.com>
发送时间: 2021年1月26日 22:07
收件人: Zhang, Qiang
抄送: Uladzislau Rezki; Paul E. McKenney; rcu@vger.kernel.org; linux-kernel@vger.kernel.org
主题: Re: 回复: 回复: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline

>
> On Fri, Jan 22, 2021 at 01:44:36AM +0000, Zhang, Qiang wrote:
> >
> >
> > ________________________________________
> > 发件人: Uladzislau Rezki <urezki@gmail.com>
> > 发送时间: 2021年1月22日 4:26
> > 收件人: Zhang, Qiang
> > 抄送: Paul E. McKenney; rcu@vger.kernel.org; linux-kernel@vger.kernel.org; urezki@gmail.com
> > 主题: Re: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
> > >Hello, Qiang,
> >
> > > On Thu, Jan 21, 2021 at 02:49:49PM +0800, qiang.zhang@windriver.com wrote:
> > > > From: Zqiang <qiang.zhang@windriver.com>
> > > >
> > > > If CPUs go offline, the corresponding krcp's page cache can
> > > > not be use util the CPU come back online, or maybe the CPU
> > > > will never go online again, this commit therefore free krcp's
> > > > page cache when CPUs go offline.
> > > >
> > > > Signed-off-by: Zqiang <qiang.zhang@windriver.com>
> > >
> > >Do you consider it as an issue? We have 5 pages per CPU, that is 20480 bytes.
> > >
> >
> > Hello Rezki
> >
> > In a multi CPUs system, more than one CPUs may be offline, there are more than 5 pages,  and these offline CPUs may never go online again  or  in the process of CPUs online, there are errors, which lead to the failure of online, these scenarios will lead to the per-cpu krc page cache will never be released.
> >
> >Thanks for your answer. I was thinking more about if you knew some >platforms
> >which suffer from such extra page usage when CPU goes offline. Any >issues
> >your platforms or devices run into because of that.
> >
> >So i understand that if CPU goes offline the 5 pages associated with it >are
> >unused until it goes online back.
>
>  I agree with you, But I still want to talk about what I think
>
>  My understanding is that when the CPU is offline,  the pages is not
>  accessible,  beacuse we don't know when this CPU will
>  go online again, so we best to return these page to the buddy system,
>  when the CPU goes online again, we can allocate page from the buddy
>  system to fill krcp's page cache.  maybe you may think that this memory
>  is small and don't need to.
>
>BTW, we can release the caches via shrinker path instead, what is more makes
>sense to me. We already have a callback, that frees pages when a page allocator
>asks for it. I think in that case it would be fair to return it to the buddy
>system. It happens under low memory condition

  I agree. it can be done in shrink callback, can release the currently existing per-cpu 
  page cache.
  
   Thanks
   Qiang
> or can be done manually to flush
>system caches:
>
>echo 3 > /proc/sys/vm/drop_caches
>
>What do you think?
>
>--
>Vlad Rezki

^ permalink raw reply	[flat|nested] 10+ messages in thread

* Re: 回复: 回复: 回复: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
  2021-01-27  9:00             ` 回复: " Zhang, Qiang
@ 2021-01-27 12:47               ` Uladzislau Rezki
  0 siblings, 0 replies; 10+ messages in thread
From: Uladzislau Rezki @ 2021-01-27 12:47 UTC (permalink / raw)
  To: Zhang, Qiang; +Cc: Uladzislau Rezki, Paul E. McKenney, rcu, linux-kernel

On Wed, Jan 27, 2021 at 09:00:27AM +0000, Zhang, Qiang wrote:
> 
> 
> ________________________________________
> 发件人: Uladzislau Rezki <urezki@gmail.com>
> 发送时间: 2021年1月26日 22:07
> 收件人: Zhang, Qiang
> 抄送: Uladzislau Rezki; Paul E. McKenney; rcu@vger.kernel.org; linux-kernel@vger.kernel.org
> 主题: Re: 回复: 回复: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
> 
> >
> > On Fri, Jan 22, 2021 at 01:44:36AM +0000, Zhang, Qiang wrote:
> > >
> > >
> > > ________________________________________
> > > 发件人: Uladzislau Rezki <urezki@gmail.com>
> > > 发送时间: 2021年1月22日 4:26
> > > 收件人: Zhang, Qiang
> > > 抄送: Paul E. McKenney; rcu@vger.kernel.org; linux-kernel@vger.kernel.org; urezki@gmail.com
> > > 主题: Re: [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline
> > > >Hello, Qiang,
> > >
> > > > On Thu, Jan 21, 2021 at 02:49:49PM +0800, qiang.zhang@windriver.com wrote:
> > > > > From: Zqiang <qiang.zhang@windriver.com>
> > > > >
> > > > > If CPUs go offline, the corresponding krcp's page cache can
> > > > > not be use util the CPU come back online, or maybe the CPU
> > > > > will never go online again, this commit therefore free krcp's
> > > > > page cache when CPUs go offline.
> > > > >
> > > > > Signed-off-by: Zqiang <qiang.zhang@windriver.com>
> > > >
> > > >Do you consider it as an issue? We have 5 pages per CPU, that is 20480 bytes.
> > > >
> > >
> > > Hello Rezki
> > >
> > > In a multi CPUs system, more than one CPUs may be offline, there are more than 5 pages,  and these offline CPUs may never go online again  or  in the process of CPUs online, there are errors, which lead to the failure of online, these scenarios will lead to the per-cpu krc page cache will never be released.
> > >
> > >Thanks for your answer. I was thinking more about if you knew some >platforms
> > >which suffer from such extra page usage when CPU goes offline. Any >issues
> > >your platforms or devices run into because of that.
> > >
> > >So i understand that if CPU goes offline the 5 pages associated with it >are
> > >unused until it goes online back.
> >
> >  I agree with you, But I still want to talk about what I think
> >
> >  My understanding is that when the CPU is offline,  the pages is not
> >  accessible,  beacuse we don't know when this CPU will
> >  go online again, so we best to return these page to the buddy system,
> >  when the CPU goes online again, we can allocate page from the buddy
> >  system to fill krcp's page cache.  maybe you may think that this memory
> >  is small and don't need to.
> >
> >BTW, we can release the caches via shrinker path instead, what is more makes
> >sense to me. We already have a callback, that frees pages when a page allocator
> >asks for it. I think in that case it would be fair to return it to the buddy
> >system. It happens under low memory condition
> 
>   I agree. it can be done in shrink callback, can release the currently existing per-cpu 
>   page cache.
>   
Would not you mind to send a patch? If you need some input, i am happy
to participate.

Thanks!

--
Vlad Rezki

^ permalink raw reply	[flat|nested] 10+ messages in thread

end of thread, other threads:[~2021-01-27 12:50 UTC | newest]

Thread overview: 10+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-01-21  6:49 [PATCH] rcu: Release per-cpu krcp page cache when CPU going offline qiang.zhang
2021-01-21 18:56 ` Paul E. McKenney
2021-01-21 20:26   ` Uladzislau Rezki
2021-01-22  1:44     ` 回复: " Zhang, Qiang
2021-01-22 14:31       ` Uladzislau Rezki
2021-01-24  2:21         ` 回复: " Zhang, Qiang
2021-01-24 17:42           ` Uladzislau Rezki
2021-01-26 14:07           ` Uladzislau Rezki
2021-01-27  9:00             ` 回复: " Zhang, Qiang
2021-01-27 12:47               ` Uladzislau Rezki

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).