[PATCH] rt-tests: pi_stress: fix testing threads' smp affinity

[PATCH] rt-tests: pi_stress: fix testing threads' smp affinity

[Jiafei Pan]

This patch includes the following modification:
1. Make sure test threads and admin threads don't run on the same CPU
   Core if uniprocessor is not set or not on single Core platform to
   avoid starve admin threads.
2. Force to use SCHED_RR if more than one Groups running one a CPU
   Core to avoid test failure because threads in different Groups
   are using the same priority, SCHED_FIFO which is default policy
   and it maybe trigger deadlock of testing threads.

Signed-off-by: Jiafei Pan <Jiafei.Pan@nxp.com>
---
 src/pi_tests/pi_stress.c | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/src/pi_tests/pi_stress.c b/src/pi_tests/pi_stress.c
index 49f89b7..8795908 100644
--- a/src/pi_tests/pi_stress.c
+++ b/src/pi_tests/pi_stress.c
@@ -237,6 +237,13 @@ int main(int argc, char **argv)
 	/* process command line arguments */
 	process_command_line(argc, argv);
 
+	if (ngroups > (num_processors - 1)) {
+		printf("Warning: One Core will used for administor thread, the other CPU Core will run test thread,\n");
+		printf("\t it will running more than one Group on one Core (groups > num_of_processors -1),\n");
+		printf("\t it will force to use SCHED_RR, or change groups number to be lower than %ld \n", num_processors);
+		policy = SCHED_RR;
+	}
+
 	/* set default sched attributes */
 	setup_sched_config(policy);
 
@@ -285,9 +292,17 @@ int main(int argc, char **argv)
 			break;
 	for (i = 0; i < ngroups; i++) {
 		groups[i].id = i;
-		groups[i].cpu = core++;
-		if (core >= num_processors)
-			core = 0;
+		if (num_processors == 1 || uniprocessor) {
+			groups[i].cpu = 0;
+		} else {
+			groups[i].cpu = core;
+			/* Find next non-admin Core */
+			do {
+				core++;
+				if (core >= num_processors)
+					core = 0;
+			} while (CPU_ISSET(core, &admin_cpu_mask));
+		}
 		if (create_group(&groups[i]) != SUCCESS)
 			return FAILURE;
 	}
@@ -1143,7 +1158,7 @@ int create_group(struct group_parameters *group)
 	CPU_ZERO(&mask);
 	CPU_SET(group->cpu, &mask);
 
-	pi_debug("group %d bound to cpu %ld\n", group->id, group->cpu);
+	printf("group %d bound to cpu %ld\n", group->id, group->cpu);
 
 	/* start the low priority thread */
 	pi_debug("creating low priority thread\n");
-- 
2.17.1

RE: [PATCH] rt-tests: pi_stress: fix testing threads' smp affinity

[Jiafei Pan]

Any comments? thanks.

Best Regards,
Jiafei.

> -----Original Message-----
> From: Jiafei Pan <jiafei.pan@nxp.com>
> Sent: Friday, March 26, 2021 5:44 PM
> To: williams@redhat.com; jkacur@redhat.com
> Cc: linux-rt-users@vger.kernel.org; Jiafei Pan <jiafei.pan@nxp.com>; Jiafei
> Pan <jiafei.pan@nxp.com>
> Subject: [PATCH] rt-tests: pi_stress: fix testing threads' smp affinity
> 
> This patch includes the following modification:
> 1. Make sure test threads and admin threads don't run on the same CPU
>    Core if uniprocessor is not set or not on single Core platform to
>    avoid starve admin threads.
> 2. Force to use SCHED_RR if more than one Groups running one a CPU
>    Core to avoid test failure because threads in different Groups
>    are using the same priority, SCHED_FIFO which is default policy
>    and it maybe trigger deadlock of testing threads.
> 
> Signed-off-by: Jiafei Pan <Jiafei.Pan@nxp.com>
> ---
>  src/pi_tests/pi_stress.c | 23 +++++++++++++++++++----
>  1 file changed, 19 insertions(+), 4 deletions(-)
> 
> diff --git a/src/pi_tests/pi_stress.c b/src/pi_tests/pi_stress.c index
> 49f89b7..8795908 100644
> --- a/src/pi_tests/pi_stress.c
> +++ b/src/pi_tests/pi_stress.c
> @@ -237,6 +237,13 @@ int main(int argc, char **argv)
>  	/* process command line arguments */
>  	process_command_line(argc, argv);
> 
> +	if (ngroups > (num_processors - 1)) {
> +		printf("Warning: One Core will used for administor thread, the other
> CPU Core will run test thread,\n");
> +		printf("\t it will running more than one Group on one Core (groups >
> num_of_processors -1),\n");
> +		printf("\t it will force to use SCHED_RR, or change groups number to
> be lower than %ld \n", num_processors);
> +		policy = SCHED_RR;
> +	}
> +
>  	/* set default sched attributes */
>  	setup_sched_config(policy);
> 
> @@ -285,9 +292,17 @@ int main(int argc, char **argv)
>  			break;
>  	for (i = 0; i < ngroups; i++) {
>  		groups[i].id = i;
> -		groups[i].cpu = core++;
> -		if (core >= num_processors)
> -			core = 0;
> +		if (num_processors == 1 || uniprocessor) {
> +			groups[i].cpu = 0;
> +		} else {
> +			groups[i].cpu = core;
> +			/* Find next non-admin Core */
> +			do {
> +				core++;
> +				if (core >= num_processors)
> +					core = 0;
> +			} while (CPU_ISSET(core, &admin_cpu_mask));
> +		}
>  		if (create_group(&groups[i]) != SUCCESS)
>  			return FAILURE;
>  	}
> @@ -1143,7 +1158,7 @@ int create_group(struct group_parameters *group)
>  	CPU_ZERO(&mask);
>  	CPU_SET(group->cpu, &mask);
> 
> -	pi_debug("group %d bound to cpu %ld\n", group->id, group->cpu);
> +	printf("group %d bound to cpu %ld\n", group->id, group->cpu);
> 
>  	/* start the low priority thread */
>  	pi_debug("creating low priority thread\n");
> --
> 2.17.1

RE: [PATCH] rt-tests: pi_stress: fix testing threads' smp affinity

[John Kacur]

On Fri, 7 May 2021, Jiafei Pan wrote:

> Any comments? thanks.
> 
> Best Regards,
> Jiafei.
> 
> > -----Original Message-----
> > From: Jiafei Pan <jiafei.pan@nxp.com>
> > Sent: Friday, March 26, 2021 5:44 PM
> > To: williams@redhat.com; jkacur@redhat.com
> > Cc: linux-rt-users@vger.kernel.org; Jiafei Pan <jiafei.pan@nxp.com>; Jiafei
> > Pan <jiafei.pan@nxp.com>
> > Subject: [PATCH] rt-tests: pi_stress: fix testing threads' smp affinity
> > 
> > This patch includes the following modification:
> > 1. Make sure test threads and admin threads don't run on the same CPU
> >    Core if uniprocessor is not set or not on single Core platform to
> >    avoid starve admin threads.

Doesn't the code already do this? The one exception I can think of is you 
are allowed one group of threads per online processor, so if you have the 
maximum number of groups, then one of those groups will run on the same 
processor as the admin thread. Even in this case though, the admin_thread
has the highest realtime priority.

> > 2. Force to use SCHED_RR if more than one Groups running one a CPU
> >    Core to avoid test failure because threads in different Groups
> >    are using the same priority, SCHED_FIFO which is default policy
> >    and it maybe trigger deadlock of testing threads.

Like a lot of things in realtime, we give you enough rope to hang 
yourself. We are trying to create a tool here that purposely triggers 
deadlocks that are resolved via priority inheritence in order to test 
priority inheritence, we're not trying to create a robust application that 
has safeguards to avoid this situation. So, with that in mind, why would 
we force the teste to use SCHED_RR if the user wants SCHED_FF ?

Trying to understand what you are trying to accomplish here. Is it that 
you want to get rid of the limitation of only allowing one inversion group 
per processor? If so, why is this useful?

Thanks

John

> > 
> > Signed-off-by: Jiafei Pan <Jiafei.Pan@nxp.com>
> > ---
> >  src/pi_tests/pi_stress.c | 23 +++++++++++++++++++----
> >  1 file changed, 19 insertions(+), 4 deletions(-)
> > 
> > diff --git a/src/pi_tests/pi_stress.c b/src/pi_tests/pi_stress.c index
> > 49f89b7..8795908 100644
> > --- a/src/pi_tests/pi_stress.c
> > +++ b/src/pi_tests/pi_stress.c
> > @@ -237,6 +237,13 @@ int main(int argc, char **argv)
> >  	/* process command line arguments */
> >  	process_command_line(argc, argv);
> > 
> > +	if (ngroups > (num_processors - 1)) {
> > +		printf("Warning: One Core will used for administor thread, the other
> > CPU Core will run test thread,\n");
> > +		printf("\t it will running more than one Group on one Core (groups >
> > num_of_processors -1),\n");
> > +		printf("\t it will force to use SCHED_RR, or change groups number to
> > be lower than %ld \n", num_processors);
> > +		policy = SCHED_RR;
> > +	}
> > +
> >  	/* set default sched attributes */
> >  	setup_sched_config(policy);
> > 
> > @@ -285,9 +292,17 @@ int main(int argc, char **argv)
> >  			break;
> >  	for (i = 0; i < ngroups; i++) {
> >  		groups[i].id = i;
> > -		groups[i].cpu = core++;
> > -		if (core >= num_processors)
> > -			core = 0;
> > +		if (num_processors == 1 || uniprocessor) {
> > +			groups[i].cpu = 0;
> > +		} else {
> > +			groups[i].cpu = core;
> > +			/* Find next non-admin Core */
> > +			do {
> > +				core++;
> > +				if (core >= num_processors)
> > +					core = 0;
> > +			} while (CPU_ISSET(core, &admin_cpu_mask));
> > +		}
> >  		if (create_group(&groups[i]) != SUCCESS)
> >  			return FAILURE;
> >  	}
> > @@ -1143,7 +1158,7 @@ int create_group(struct group_parameters *group)
> >  	CPU_ZERO(&mask);
> >  	CPU_SET(group->cpu, &mask);
> > 
> > -	pi_debug("group %d bound to cpu %ld\n", group->id, group->cpu);
> > +	printf("group %d bound to cpu %ld\n", group->id, group->cpu);
> > 
> >  	/* start the low priority thread */
> >  	pi_debug("creating low priority thread\n");
> > --
> > 2.17.1
> 
> 

RE: [EXT] RE: [PATCH] rt-tests: pi_stress: fix testing threads' smp affinity

[Jiafei Pan]

> -----Original Message-----
> From: John Kacur <jkacur@gmail.com> On Behalf Of John Kacur
> Sent: Saturday, May 8, 2021 12:02 AM
> To: Jiafei Pan <jiafei.pan@nxp.com>
> Cc: williams@redhat.com; linux-rt-users@vger.kernel.org
> Subject: [EXT] RE: [PATCH] rt-tests: pi_stress: fix testing threads' smp affinity
> 
> Caution: EXT Email
> 
> On Fri, 7 May 2021, Jiafei Pan wrote:
> 
> > Any comments? thanks.
> >
> > Best Regards,
> > Jiafei.
> >
> > > -----Original Message-----
> > > From: Jiafei Pan <jiafei.pan@nxp.com>
> > > Sent: Friday, March 26, 2021 5:44 PM
> > > To: williams@redhat.com; jkacur@redhat.com
> > > Cc: linux-rt-users@vger.kernel.org; Jiafei Pan <jiafei.pan@nxp.com>;
> > > Jiafei Pan <jiafei.pan@nxp.com>
> > > Subject: [PATCH] rt-tests: pi_stress: fix testing threads' smp
> > > affinity
> > >
> > > This patch includes the following modification:
> > > 1. Make sure test threads and admin threads don't run on the same CPU
> > >    Core if uniprocessor is not set or not on single Core platform to
> > >    avoid starve admin threads.
> 
> Doesn't the code already do this? The one exception I can think of is you are
> allowed one group of threads per online processor, so if you have the
> maximum number of groups, then one of those groups will run on the same
> processor as the admin thread. Even in this case though, the admin_thread
> has the highest realtime priority.

Yes, you are right, but I am not sure why system hangs while running pi_stress
with 2 groups on 2 CPU Cores platform, but it has no hang issue only run 1 group
on the same platform. Will dig more.

Thanks,
Jiafei.

> 
> > > 2. Force to use SCHED_RR if more than one Groups running one a CPU
> > >    Core to avoid test failure because threads in different Groups
> > >    are using the same priority, SCHED_FIFO which is default policy
> > >    and it maybe trigger deadlock of testing threads.
> 
> Like a lot of things in realtime, we give you enough rope to hang yourself. We
> are trying to create a tool here that purposely triggers deadlocks that are
> resolved via priority inheritence in order to test priority inheritence, we're not
> trying to create a robust application that has safeguards to avoid this situation.
> So, with that in mind, why would we force the teste to use SCHED_RR if the
> user wants SCHED_FF ?
> 
> Trying to understand what you are trying to accomplish here. Is it that you
> want to get rid of the limitation of only allowing one inversion group per
> processor? If so, why is this useful?
> 
> Thanks
> 
> John

I want to limit only one group per process, not sure whether there is some
Issue or not when run two or more groups on the same process because
They have the same priorities for each group.

Thanks.
Jiafei.

> 
> > >
> > > Signed-off-by: Jiafei Pan <Jiafei.Pan@nxp.com>
> > > ---
> > >  src/pi_tests/pi_stress.c | 23 +++++++++++++++++++----
> > >  1 file changed, 19 insertions(+), 4 deletions(-)
> > >
> > > diff --git a/src/pi_tests/pi_stress.c b/src/pi_tests/pi_stress.c
> > > index
> > > 49f89b7..8795908 100644
> > > --- a/src/pi_tests/pi_stress.c
> > > +++ b/src/pi_tests/pi_stress.c
> > > @@ -237,6 +237,13 @@ int main(int argc, char **argv)
> > >     /* process command line arguments */
> > >     process_command_line(argc, argv);
> > >
> > > +   if (ngroups > (num_processors - 1)) {
> > > +           printf("Warning: One Core will used for administor
> > > + thread, the other
> > > CPU Core will run test thread,\n");
> > > +           printf("\t it will running more than one Group on one
> > > + Core (groups >
> > > num_of_processors -1),\n");
> > > +           printf("\t it will force to use SCHED_RR, or change
> > > + groups number to
> > > be lower than %ld \n", num_processors);
> > > +           policy = SCHED_RR;
> > > +   }
> > > +
> > >     /* set default sched attributes */
> > >     setup_sched_config(policy);
> > >
> > > @@ -285,9 +292,17 @@ int main(int argc, char **argv)
> > >                     break;
> > >     for (i = 0; i < ngroups; i++) {
> > >             groups[i].id = i;
> > > -           groups[i].cpu = core++;
> > > -           if (core >= num_processors)
> > > -                   core = 0;
> > > +           if (num_processors == 1 || uniprocessor) {
> > > +                   groups[i].cpu = 0;
> > > +           } else {
> > > +                   groups[i].cpu = core;
> > > +                   /* Find next non-admin Core */
> > > +                   do {
> > > +                           core++;
> > > +                           if (core >= num_processors)
> > > +                                   core = 0;
> > > +                   } while (CPU_ISSET(core, &admin_cpu_mask));
> > > +           }
> > >             if (create_group(&groups[i]) != SUCCESS)
> > >                     return FAILURE;
> > >     }
> > > @@ -1143,7 +1158,7 @@ int create_group(struct group_parameters
> *group)
> > >     CPU_ZERO(&mask);
> > >     CPU_SET(group->cpu, &mask);
> > >
> > > -   pi_debug("group %d bound to cpu %ld\n", group->id, group->cpu);
> > > +   printf("group %d bound to cpu %ld\n", group->id, group->cpu);
> > >
> > >     /* start the low priority thread */
> > >     pi_debug("creating low priority thread\n");
> > > --
> > > 2.17.1
> >
> >