[LTP] [PATCH] memcg_stress_test.sh: fix reserved mem calculate

[LTP] [PATCH] memcg_stress_test.sh: fix reserved mem calculate

[Hao Ge]

When running this test case on a machine with large memory,
and without swap or swap is too small,existing reserved
memory is too small for a machine with large memory,and
will cause forking a subprocess to run a command will fail
due to memory is exhausted,so optimize reserved memory
calculate to ten percent of free memory.

Here is an example:

free -m
              total        used        free      shared  buff/cache   available
Mem:         260184        2959      255854          62        1370      236346
Swap:          4095           0        4095

Signed-off-by: Hao Ge <gehao@kylinos.cn>
---
 .../kernel/controllers/memcg/stress/memcg_stress_test.sh    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh b/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
index cb52840d7..f26a9f72a 100755
--- a/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
+++ b/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
@@ -40,7 +40,7 @@ setup()
 	MEM=$(( $mem_free + $swap_free / 2 ))
 	MEM=$(( $MEM / 1024 ))
 	RUN_TIME=$(( 15 * 60 ))
-	[ "$pgsize" = "4096" ] && THREAD_SPARE_MB=1 || THREAD_SPARE_MB=8
+	RESERVED_MEMORY=$(( $MEM * 10/100 ))
 
 	tst_res TINFO "Calculated available memory $MEM MB"
 }
@@ -93,12 +93,12 @@ run_stress()
 
 test1()
 {
-	run_stress 150 $(( ($MEM - 150 * $THREAD_SPARE_MB) / 150 )) 5 $RUN_TIME
+	run_stress 150 $(( ($MEM - $RESERVED_MEMORY) / 150 )) 5 $RUN_TIME
 }
 
 test2()
 {
-	run_stress 1 $(( $MEM - $THREAD_SPARE_MB)) 5 $RUN_TIME
+	run_stress 1 $(( $MEM - $RESERVED_MEMORY)) 5 $RUN_TIME
 }
 
 . cgroup_lib.sh
-- 
2.25.1


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

[LTP] [PATCH V2] memcg_stress_test.sh: Fix reserved mem calculate

[Hao Ge]

When running this test case on a machine with large memory,
and without swap or swap is too small,existing reserved
memory is too small for a machine with large memory,and
will cause forking a subprocess to run a command will fail
due to memory is exhausted,so optimize reserved memory
calculate to ten percent of free memory.

Here is an example:

free -m
              total        used        free      shared  buff/cache   available
Mem:         260184        2959      255854          62        1370      236346
Swap:          4095           0        4095

when you run this testcase,console will report
fork：Cannot allocate memory

Signed-off-by: Hao Ge <gehao@kylinos.cn>
---
 .../kernel/controllers/memcg/stress/memcg_stress_test.sh    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh b/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
index cb52840d7..f26a9f72a 100755
--- a/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
+++ b/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
@@ -40,7 +40,7 @@ setup()
 	MEM=$(( $mem_free + $swap_free / 2 ))
 	MEM=$(( $MEM / 1024 ))
 	RUN_TIME=$(( 15 * 60 ))
-	[ "$pgsize" = "4096" ] && THREAD_SPARE_MB=1 || THREAD_SPARE_MB=8
+	RESERVED_MEMORY=$(( $MEM * 10/100 ))
 
 	tst_res TINFO "Calculated available memory $MEM MB"
 }
@@ -93,12 +93,12 @@ run_stress()
 
 test1()
 {
-	run_stress 150 $(( ($MEM - 150 * $THREAD_SPARE_MB) / 150 )) 5 $RUN_TIME
+	run_stress 150 $(( ($MEM - $RESERVED_MEMORY) / 150 )) 5 $RUN_TIME
 }
 
 test2()
 {
-	run_stress 1 $(( $MEM - $THREAD_SPARE_MB)) 5 $RUN_TIME
+	run_stress 1 $(( $MEM - $RESERVED_MEMORY)) 5 $RUN_TIME
 }
 
 . cgroup_lib.sh
-- 
2.25.1


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

Re: [LTP] [PATCH V2] memcg_stress_test.sh: Fix reserved mem calculate

[gehao618]

Hi Pals


    Can you help to review this patch ? Or have any suggestions ?


Thanks
Best Regards
Hao
---- Replied Message ----
| From | Hao Ge<gehao@kylinos.cn> |
| Date | 03/09/2023 17:16 |
| To | ltp<ltp@lists.linux.it> |
| Cc | Hao Ge<gehao@kylinos.cn> |
| Subject | [LTP] [PATCH V2] memcg_stress_test.sh: Fix reserved mem calculate |
When running this test case on a machine with large memory,
and without swap or swap is too small,existing reserved
memory is too small for a machine with large memory,and
will cause forking a subprocess to run a command will fail
due to memory is exhausted,so optimize reserved memory
calculate to ten percent of free memory.

Here is an example:

free -m
             total        used        free      shared  buff/cache   available
Mem:         260184        2959      255854          62        1370      236346
Swap:          4095           0        4095

when you run this testcase,console will report
fork：Cannot allocate memory

Signed-off-by: Hao Ge <gehao@kylinos.cn>
---
.../kernel/controllers/memcg/stress/memcg_stress_test.sh    | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh b/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
index cb52840d7..f26a9f72a 100755
--- a/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
+++ b/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
@@ -40,7 +40,7 @@ setup()
   MEM=$(( $mem_free + $swap_free / 2 ))
   MEM=$(( $MEM / 1024 ))
   RUN_TIME=$(( 15 * 60 ))
-    [ "$pgsize" = "4096" ] && THREAD_SPARE_MB=1 || THREAD_SPARE_MB=8
+    RESERVED_MEMORY=$(( $MEM * 10/100 ))

   tst_res TINFO "Calculated available memory $MEM MB"
}
@@ -93,12 +93,12 @@ run_stress()

test1()
{
-    run_stress 150 $(( ($MEM - 150 * $THREAD_SPARE_MB) / 150 )) 5 $RUN_TIME
+    run_stress 150 $(( ($MEM - $RESERVED_MEMORY) / 150 )) 5 $RUN_TIME
}

test2()
{
-    run_stress 1 $(( $MEM - $THREAD_SPARE_MB)) 5 $RUN_TIME
+    run_stress 1 $(( $MEM - $RESERVED_MEMORY)) 5 $RUN_TIME
}

. cgroup_lib.sh
--
2.25.1


--
Mailing list info: https://lists.linux.it/listinfo/ltp

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

Re: [LTP] [PATCH V2] memcg_stress_test.sh: Fix reserved mem calculate

[Li Wang]

Hi Hao,

On Thu, Mar 9, 2023 at 5:17 PM Hao Ge <gehao@kylinos.cn> wrote:

> When running this test case on a machine with large memory,
> and without swap or swap is too small,existing reserved
> memory is too small for a machine with large memory,and
> will cause forking a subprocess to run a command will fail
> due to memory is exhausted,so optimize reserved memory
> calculate to ten percent of free memory.
>
> Here is an example:
>
> free -m
>               total        used        free      shared  buff/cache
>  available
> Mem:         260184        2959      255854          62        1370
> 236346
> Swap:          4095           0        4095
>
> when you run this testcase,console will report
> fork：Cannot allocate memory
>
> Signed-off-by: Hao Ge <gehao@kylinos.cn>
> ---
>  .../kernel/controllers/memcg/stress/memcg_stress_test.sh    | 6 +++---
>  1 file changed, 3 insertions(+), 3 deletions(-)
>
> diff --git
> a/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
> b/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
> index cb52840d7..f26a9f72a 100755
> --- a/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
> +++ b/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
> @@ -40,7 +40,7 @@ setup()
>         MEM=$(( $mem_free + $swap_free / 2 ))
>         MEM=$(( $MEM / 1024 ))
>         RUN_TIME=$(( 15 * 60 ))
> -       [ "$pgsize" = "4096" ] && THREAD_SPARE_MB=1 || THREAD_SPARE_MB=8
> +       RESERVED_MEMORY=$(( $MEM * 10/100 ))
>

Thanks for the improvement work.

But IMHO, this does not completely resolve the potential issue which
in eating free_mem for stress testing.

We have to consider many scenarios for the system's available memory
including without swap file, free_mem > avail_mem, and min_free_kbytes
impaction.

For example, your patch may _not_ work on this:

MemTotal:       36044544 kB
MemFree:        29177728 kB
MemAvailable:   26391488 kB
...
SwapTotal:             0 kB
SwapFree:              0 kB

I would suggest you take a look into:

https://github.com/linux-test-project/ltp/blob/master/lib/tst_memutils.c#L19
to see how this function treats this and keeps a safety mem margin.
Hope it gives you some inspiration.




>
>         tst_res TINFO "Calculated available memory $MEM MB"
>  }
> @@ -93,12 +93,12 @@ run_stress()
>
>  test1()
>  {
> -       run_stress 150 $(( ($MEM - 150 * $THREAD_SPARE_MB) / 150 )) 5
> $RUN_TIME
> +       run_stress 150 $(( ($MEM - $RESERVED_MEMORY) / 150 )) 5 $RUN_TIME
>  }
>
>  test2()
>  {
> -       run_stress 1 $(( $MEM - $THREAD_SPARE_MB)) 5 $RUN_TIME
> +       run_stress 1 $(( $MEM - $RESERVED_MEMORY)) 5 $RUN_TIME
>  }
>
>  . cgroup_lib.sh
> --
> 2.25.1
>
>
> --
> Mailing list info: https://lists.linux.it/listinfo/ltp
>


-- 
Regards,
Li Wang

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

Re: [LTP] [PATCH V2] memcg_stress_test.sh: Fix reserved mem calculate

[gehao618]

Hi Li


    Thank you for pointing this out.
    I get a lot of inspiration form it.
    These calculations are more accurate.


Thanks
Best Regards
Hao
---- Replied Message ----
| From | Li Wang<liwang@redhat.com> |
| Date | 03/13/2023 13:59 |
| To | Hao Ge<gehao@kylinos.cn> |
| Cc | ltp<ltp@lists.linux.it> |
| Subject | Re: [LTP] [PATCH V2] memcg_stress_test.sh: Fix reserved mem calculate |
Hi Hao,

On Thu, Mar 9, 2023 at 5:17 PM Hao Ge <gehao@kylinos.cn> wrote:

> When running this test case on a machine with large memory,
> and without swap or swap is too small,existing reserved
> memory is too small for a machine with large memory,and
> will cause forking a subprocess to run a command will fail
> due to memory is exhausted,so optimize reserved memory
> calculate to ten percent of free memory.
>
> Here is an example:
>
> free -m
>               total        used        free      shared  buff/cache
>  available
> Mem:         260184        2959      255854          62        1370
> 236346
> Swap:          4095           0        4095
>
> when you run this testcase,console will report
> fork：Cannot allocate memory
>
> Signed-off-by: Hao Ge <gehao@kylinos.cn>
> ---
>  .../kernel/controllers/memcg/stress/memcg_stress_test.sh    | 6 +++---
>  1 file changed, 3 insertions(+), 3 deletions(-)
>
> diff --git
> a/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
> b/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
> index cb52840d7..f26a9f72a 100755
> --- a/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
> +++ b/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
> @@ -40,7 +40,7 @@ setup()
>         MEM=$(( $mem_free + $swap_free / 2 ))
>         MEM=$(( $MEM / 1024 ))
>         RUN_TIME=$(( 15 * 60 ))
> -       [ "$pgsize" = "4096" ] && THREAD_SPARE_MB=1 || THREAD_SPARE_MB=8
> +       RESERVED_MEMORY=$(( $MEM * 10/100 ))
>

Thanks for the improvement work.

But IMHO, this does not completely resolve the potential issue which
in eating free_mem for stress testing.

We have to consider many scenarios for the system's available memory
including without swap file, free_mem > avail_mem, and min_free_kbytes
impaction.

For example, your patch may _not_ work on this:

MemTotal:       36044544 kB
MemFree:        29177728 kB
MemAvailable:   26391488 kB
...
SwapTotal:             0 kB
SwapFree:              0 kB

I would suggest you take a look into:

https://github.com/linux-test-project/ltp/blob/master/lib/tst_memutils.c#L19
to see how this function treats this and keeps a safety mem margin.
Hope it gives you some inspiration.




>
>         tst_res TINFO "Calculated available memory $MEM MB"
>  }
> @@ -93,12 +93,12 @@ run_stress()
>
>  test1()
>  {
> -       run_stress 150 $(( ($MEM - 150 * $THREAD_SPARE_MB) / 150 )) 5
> $RUN_TIME
> +       run_stress 150 $(( ($MEM - $RESERVED_MEMORY) / 150 )) 5 $RUN_TIME
>  }
>
>  test2()
>  {
> -       run_stress 1 $(( $MEM - $THREAD_SPARE_MB)) 5 $RUN_TIME
> +       run_stress 1 $(( $MEM - $RESERVED_MEMORY)) 5 $RUN_TIME
>  }
>
>  . cgroup_lib.sh
> --
> 2.25.1
>
>
> --
> Mailing list info: https://lists.linux.it/listinfo/ltp
>


--
Regards,
Li Wang

--
Mailing list info: https://lists.linux.it/listinfo/ltp

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

[LTP] [PATCH V3] memcg_stress_test.sh: Fix reserved mem calculate

[Hao Ge]

When running this test case on a machine with large memory,
and without swap or swap is too small,existing reserved
memory is too small for a machine with large memory,and
will cause forking a subprocess to run a command will fail
due to memory is exhausted,so optimize reserved memory
calculate .

Signed-off-by: Hao Ge <gehao@kylinos.cn>
---
 .../memcg/stress/memcg_stress_test.sh         | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh b/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
index cb52840d7..2c0488170 100755
--- a/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
+++ b/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
@@ -34,13 +34,20 @@ setup()
 	echo 3 > /proc/sys/vm/drop_caches
 	sleep 2
 	local mem_free=`cat /proc/meminfo | grep MemFree | awk '{ print $2 }'`
+	local mem_available=`cat /proc/meminfo | grep MemAvailable | awk '{ print $2 }'`
 	local swap_free=`cat /proc/meminfo | grep SwapFree | awk '{ print $2 }'`
-	local pgsize=`tst_getconf PAGESIZE`
-
-	MEM=$(( $mem_free + $swap_free / 2 ))
+	mem_min=`cat /proc/sys/vm/min_free_kbytes`
+
+	#Apply a margin because we cannot get below "min" watermark
+	mem_min=$(( $mem_min + $mem_min/10 ))
+	#we need calculate RESERVED_MEN , if have enough swap ,RESERVED_MEM =0
+	#else RESERVED_MEN = mem_min + mem_min/10
+	[ $swap_free -gt $mem_min ] && RESERVED_MEM=0 || RESERVED_MEM=$mem_min
+	# Use the lower value of free and available to calculate MEM
+	[ $mem_free -lt $mem_available ] && MEM=$mem_free || MEM=$mem_available
+	MEM=$(( $MEM - $RESERVED_MEM ))
 	MEM=$(( $MEM / 1024 ))
 	RUN_TIME=$(( 15 * 60 ))
-	[ "$pgsize" = "4096" ] && THREAD_SPARE_MB=1 || THREAD_SPARE_MB=8
 
 	tst_res TINFO "Calculated available memory $MEM MB"
 }
@@ -93,12 +100,12 @@ run_stress()
 
 test1()
 {
-	run_stress 150 $(( ($MEM - 150 * $THREAD_SPARE_MB) / 150 )) 5 $RUN_TIME
+	run_stress 150 $(( $MEM  / 150 )) 5 $RUN_TIME
 }
 
 test2()
 {
-	run_stress 1 $(( $MEM - $THREAD_SPARE_MB)) 5 $RUN_TIME
+	run_stress 1 $MEM 5 $RUN_TIME
 }
 
 . cgroup_lib.sh
-- 
2.25.1


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

[LTP] [PATCH v4] memcg_stress_test.sh: Fix reserved mem calculate

[Hao Ge]

When running this test case on a machine with large memory,
and without swap or swap is too small,existing reserved
memory is too small for a machine with large memory,and
will cause forking a subprocess to run a command will fail
due to memory is exhausted,so optimize reserved memory
calculate .

Signed-off-by: Hao Ge <gehao@kylinos.cn>
---
 .../controllers/memcg/stress/memcg_stress_test.sh   | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh b/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
index cb52840d7..ed35eaace 100755
--- a/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
+++ b/testcases/kernel/controllers/memcg/stress/memcg_stress_test.sh
@@ -34,13 +34,16 @@ setup()
 	echo 3 > /proc/sys/vm/drop_caches
 	sleep 2
 	local mem_free=`cat /proc/meminfo | grep MemFree | awk '{ print $2 }'`
+	local mem_available=`cat /proc/meminfo | grep MemAvailable | awk '{ print $2 }'`
 	local swap_free=`cat /proc/meminfo | grep SwapFree | awk '{ print $2 }'`
-	local pgsize=`tst_getconf PAGESIZE`
+	mem_min=`cat /proc/sys/vm/min_free_kbytes`
 
-	MEM=$(( $mem_free + $swap_free / 2 ))
+	mem_min=$(( $mem_min + $mem_min / 10 ))
+	[ $swap_free -gt $mem_min ] && RESERVED_MEM=0 || RESERVED_MEM=$mem_min
+	[ $mem_free -lt $mem_available ] && MEM=$mem_free || MEM=$mem_available
+	MEM=$(( $MEM - $RESERVED_MEM ))
 	MEM=$(( $MEM / 1024 ))
 	RUN_TIME=$(( 15 * 60 ))
-	[ "$pgsize" = "4096" ] && THREAD_SPARE_MB=1 || THREAD_SPARE_MB=8
 
 	tst_res TINFO "Calculated available memory $MEM MB"
 }
@@ -93,12 +96,12 @@ run_stress()
 
 test1()
 {
-	run_stress 150 $(( ($MEM - 150 * $THREAD_SPARE_MB) / 150 )) 5 $RUN_TIME
+	run_stress 150 $(( $MEM  / 150 )) 5 $RUN_TIME
 }
 
 test2()
 {
-	run_stress 1 $(( $MEM - $THREAD_SPARE_MB)) 5 $RUN_TIME
+	run_stress 1 $MEM 5 $RUN_TIME
 }
 
 . cgroup_lib.sh
-- 
2.25.1


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

Re: [LTP] [PATCH v4] memcg_stress_test.sh: Fix reserved mem calculate

[Li Wang]

Hi Hao,

I adjust some syntax and expressions then pushed. Thanks!

-- 
Regards,
Li Wang

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp