[PATCH] perf/record: add num-synthesize-threads option

[PATCH] perf/record: add num-synthesize-threads option

[Ian Rogers]

From: Stephane Eranian <eranian@google.com>

To control degree of parallelism of the synthesize_mmap() code which
is scanning /proc/PID/task/PID/maps and can be time consuming.
Mimic perf top way of handling the option.
If not specified will default to 1 thread, i.e. default behavior before
this option.

Signed-off-by: Stephane Eranian <eranian@google.com>
Reviewed-by: Ian Rogers <irogers@google.com>
---
 tools/perf/Documentation/perf-record.txt |  4 +++
 tools/perf/builtin-record.c              | 35 ++++++++++++++++++++++--
 tools/perf/util/record.h                 |  1 +
 3 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt
index b3f3b3f1c161..6e8b4649307c 100644
--- a/tools/perf/Documentation/perf-record.txt
+++ b/tools/perf/Documentation/perf-record.txt
@@ -596,6 +596,10 @@ Make a copy of /proc/kcore and place it into a directory with the perf data file
 Limit the sample data max size, <size> is expected to be a number with
 appended unit character - B/K/M/G
 
+--num-thread-synthesize::
+	The number of threads to run when synthesizing events for existing processes.
+	By default, the number of threads equals 1.
+
 SEE ALSO
 --------
 linkperf:perf-stat[1], linkperf:perf-list[1], linkperf:perf-intel-pt[1]
diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
index 1ab349abe904..2f97d0c32a75 100644
--- a/tools/perf/builtin-record.c
+++ b/tools/perf/builtin-record.c
@@ -43,6 +43,7 @@
 #include "util/time-utils.h"
 #include "util/units.h"
 #include "util/bpf-event.h"
+#include "util/util.h"
 #include "asm/bug.h"
 #include "perf.h"
 
@@ -50,6 +51,7 @@
 #include <inttypes.h>
 #include <locale.h>
 #include <poll.h>
+#include <pthread.h>
 #include <unistd.h>
 #include <sched.h>
 #include <signal.h>
@@ -503,6 +505,20 @@ static int process_synthesized_event(struct perf_tool *tool,
 	return record__write(rec, NULL, event, event->header.size);
 }
 
+static int process_locked_synthesized_event(struct perf_tool *tool,
+				     union perf_event *event,
+				     struct perf_sample *sample __maybe_unused,
+				     struct machine *machine __maybe_unused)
+{
+	static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER;
+	int ret;
+
+	pthread_mutex_lock(&synth_lock);
+	ret = process_synthesized_event(tool, event, sample, machine);
+	pthread_mutex_unlock(&synth_lock);
+	return ret;
+}
+
 static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
 {
 	struct record *rec = to;
@@ -1288,6 +1304,8 @@ static int record__synthesize(struct record *rec, bool tail)
 	struct perf_tool *tool = &rec->tool;
 	int fd = perf_data__fd(data);
 	int err = 0;
+	int (*f)(struct perf_tool *, union perf_event *, struct perf_sample *,
+		struct machine *) = process_synthesized_event;
 
 	if (rec->opts.tail_synthesize != tail)
 		return 0;
@@ -1402,9 +1420,18 @@ static int record__synthesize(struct record *rec, bool tail)
 	if (err < 0)
 		pr_warning("Couldn't synthesize cgroup events.\n");
 
+	if (rec->opts.nr_threads_synthesize > 1) {
+		perf_set_multithreaded();
+		f = process_locked_synthesized_event;
+	}
+
 	err = __machine__synthesize_threads(machine, tool, &opts->target, rec->evlist->core.threads,
-					    process_synthesized_event, opts->sample_address,
-					    1);
+					    f, opts->sample_address,
+					    rec->opts.nr_threads_synthesize);
+
+	if (rec->opts.nr_threads_synthesize > 1)
+		perf_set_singlethreaded();
+
 out:
 	return err;
 }
@@ -2232,6 +2259,7 @@ static struct record record = {
 			.default_per_cpu = true,
 		},
 		.mmap_flush          = MMAP_FLUSH_DEFAULT,
+		.nr_threads_synthesize = 1,
 	},
 	.tool = {
 		.sample		= process_sample_event,
@@ -2421,6 +2449,9 @@ static struct option __record_options[] = {
 #endif
 	OPT_CALLBACK(0, "max-size", &record.output_max_size,
 		     "size", "Limit the maximum size of the output file", parse_output_max_size),
+	OPT_UINTEGER(0, "num-thread-synthesize",
+		     &record.opts.nr_threads_synthesize,
+		     "number of threads to run for event synthesis"),
 	OPT_END()
 };
 
diff --git a/tools/perf/util/record.h b/tools/perf/util/record.h
index 24316458be20..923565c3b155 100644
--- a/tools/perf/util/record.h
+++ b/tools/perf/util/record.h
@@ -68,6 +68,7 @@ struct record_opts {
 	int	      affinity;
 	int	      mmap_flush;
 	unsigned int  comp_level;
+	unsigned int  nr_threads_synthesize;
 };
 
 extern const char * const *record_usage;
-- 
2.26.0.110.g2183baf09c-goog

Re: [PATCH] perf/record: add num-synthesize-threads option

[Jiri Olsa]

On Wed, Apr 15, 2020 at 05:13:03PM -0700, Ian Rogers wrote:

SNIP

> diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
> index 1ab349abe904..2f97d0c32a75 100644
> --- a/tools/perf/builtin-record.c
> +++ b/tools/perf/builtin-record.c
> @@ -43,6 +43,7 @@
>  #include "util/time-utils.h"
>  #include "util/units.h"
>  #include "util/bpf-event.h"
> +#include "util/util.h"
>  #include "asm/bug.h"
>  #include "perf.h"
>  
> @@ -50,6 +51,7 @@
>  #include <inttypes.h>
>  #include <locale.h>
>  #include <poll.h>
> +#include <pthread.h>
>  #include <unistd.h>
>  #include <sched.h>
>  #include <signal.h>
> @@ -503,6 +505,20 @@ static int process_synthesized_event(struct perf_tool *tool,
>  	return record__write(rec, NULL, event, event->header.size);
>  }
>  
> +static int process_locked_synthesized_event(struct perf_tool *tool,
> +				     union perf_event *event,
> +				     struct perf_sample *sample __maybe_unused,
> +				     struct machine *machine __maybe_unused)
> +{
> +	static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER;
> +	int ret;
> +
> +	pthread_mutex_lock(&synth_lock);
> +	ret = process_synthesized_event(tool, event, sample, machine);
> +	pthread_mutex_unlock(&synth_lock);

hum, so how much faster is the synthesizing with threads in record,
given that we serialize it on every event that goes to the file?

> +	return ret;
> +}
> +
>  static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
>  {
>  	struct record *rec = to;
> @@ -1288,6 +1304,8 @@ static int record__synthesize(struct record *rec, bool tail)
>  	struct perf_tool *tool = &rec->tool;
>  	int fd = perf_data__fd(data);
>  	int err = 0;
> +	int (*f)(struct perf_tool *, union perf_event *, struct perf_sample *,
> +		struct machine *) = process_synthesized_event;

there's event_op typedef in util/tools.h

jirka

Re: [PATCH] perf/record: add num-synthesize-threads option

[Ian Rogers]

On Mon, Apr 20, 2020 at 1:35 AM Jiri Olsa <jolsa@redhat.com> wrote:
>
> On Wed, Apr 15, 2020 at 05:13:03PM -0700, Ian Rogers wrote:
>
> SNIP
>
> > diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
> > index 1ab349abe904..2f97d0c32a75 100644
> > --- a/tools/perf/builtin-record.c
> > +++ b/tools/perf/builtin-record.c
> > @@ -43,6 +43,7 @@
> >  #include "util/time-utils.h"
> >  #include "util/units.h"
> >  #include "util/bpf-event.h"
> > +#include "util/util.h"
> >  #include "asm/bug.h"
> >  #include "perf.h"
> >
> > @@ -50,6 +51,7 @@
> >  #include <inttypes.h>
> >  #include <locale.h>
> >  #include <poll.h>
> > +#include <pthread.h>
> >  #include <unistd.h>
> >  #include <sched.h>
> >  #include <signal.h>
> > @@ -503,6 +505,20 @@ static int process_synthesized_event(struct perf_tool *tool,
> >       return record__write(rec, NULL, event, event->header.size);
> >  }
> >
> > +static int process_locked_synthesized_event(struct perf_tool *tool,
> > +                                  union perf_event *event,
> > +                                  struct perf_sample *sample __maybe_unused,
> > +                                  struct machine *machine __maybe_unused)
> > +{
> > +     static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER;
> > +     int ret;
> > +
> > +     pthread_mutex_lock(&synth_lock);
> > +     ret = process_synthesized_event(tool, event, sample, machine);
> > +     pthread_mutex_unlock(&synth_lock);
>
> hum, so how much faster is the synthesizing with threads in record,
> given that we serialize it on every event that goes to the file?

We see long synthesis times of the order seconds on loaded >100 core
servers. I've not been able to create a reproduction on my desktop.
You are right that making synthesis multithreaded will suffer from
Amdahl's law if the write is a synchronization point. Measuring with
the following patch in place:
https://lore.kernel.org/lkml/20200415054050.31645-4-irogers@google.com/
without threads the portion that needs a lock is less than 1.5% of
execution time and so there's plenty to still run in parallel:
...
      - 32.59% __perf_event__synthesize_threads
         - 32.54% __event__synthesize_thread
            + 22.13% perf_event__synthesize_mmap_events
            + 6.68% perf_event__get_comm_ids.constprop.0
            + 1.49% process_synthesized_event
            + 1.29% __GI___readdir64
            + 0.60% __opendir
...

The multi-threaded benchmark in this patch (pass -t):
https://lore.kernel.org/lkml/20200415054050.31645-2-irogers@google.com/
shows:

Computing performance of multi threaded perf event synthesis by
synthesizing events on CPU 0:
 Number of synthesis threads: 1
   Average synthesis took: 127729.000 usec (+- 3372.880 usec)
   Average num. events: 21548.600 (+- 0.306)
   Average time per event 5.927 usec
 Number of synthesis threads: 2
   Average synthesis took: 88863.500 usec (+- 385.168 usec)
   Average num. events: 21552.800 (+- 0.327)
   Average time per event 4.123 usec
 Number of synthesis threads: 3
   Average synthesis took: 83257.400 usec (+- 348.617 usec)
   Average num. events: 21553.200 (+- 0.327)
   Average time per event 3.863 usec
 Number of synthesis threads: 4
   Average synthesis took: 75093.000 usec (+- 422.978 usec)
   Average num. events: 21554.200 (+- 0.200)
   Average time per event 3.484 usec
 Number of synthesis threads: 5
   Average synthesis took: 64896.600 usec (+- 353.348 usec)
   Average num. events: 21558.000 (+- 0.000)
   Average time per event 3.010 usec
 Number of synthesis threads: 6
   Average synthesis took: 59210.200 usec (+- 342.890 usec)
   Average num. events: 21560.000 (+- 0.000)
   Average time per event 2.746 usec
 Number of synthesis threads: 7
   Average synthesis took: 54093.900 usec (+- 306.247 usec)
   Average num. events: 21562.000 (+- 0.000)
   Average time per event 2.509 usec
 Number of synthesis threads: 8
   Average synthesis took: 48938.700 usec (+- 341.732 usec)
   Average num. events: 21564.000 (+- 0.000)
   Average time per event 2.269 usec

The event logic there is using an atomic rather than a lock and the
scaling isn't linear as not all the logic is threaded. Still with 8
threads we see things going about 2.6 times faster. On a large loaded
machine that may bring 10 seconds of event synthesis down to less than
4. On a desktop there's no measurable difference and the
--num-thread-synthesize is defaulted to 1.

> > +     return ret;
> > +}
> > +
> >  static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
> >  {
> >       struct record *rec = to;
> > @@ -1288,6 +1304,8 @@ static int record__synthesize(struct record *rec, bool tail)
> >       struct perf_tool *tool = &rec->tool;
> >       int fd = perf_data__fd(data);
> >       int err = 0;
> > +     int (*f)(struct perf_tool *, union perf_event *, struct perf_sample *,
> > +             struct machine *) = process_synthesized_event;
>
> there's event_op typedef in util/tools.h

Thanks! Updated here:
https://lore.kernel.org/lkml/20200421003020.37611-1-irogers@google.com/

Ian

> jirka
>

Re: [PATCH] perf/record: add num-synthesize-threads option

[Jiri Olsa]

On Mon, Apr 20, 2020 at 05:31:41PM -0700, Ian Rogers wrote:

SNIP

> > > +{
> > > +     static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER;
> > > +     int ret;
> > > +
> > > +     pthread_mutex_lock(&synth_lock);
> > > +     ret = process_synthesized_event(tool, event, sample, machine);
> > > +     pthread_mutex_unlock(&synth_lock);
> >
> > hum, so how much faster is the synthesizing with threads in record,
> > given that we serialize it on every event that goes to the file?
> 
> We see long synthesis times of the order seconds on loaded >100 core
> servers. I've not been able to create a reproduction on my desktop.
> You are right that making synthesis multithreaded will suffer from
> Amdahl's law if the write is a synchronization point. Measuring with
> the following patch in place:
> https://lore.kernel.org/lkml/20200415054050.31645-4-irogers@google.com/
> without threads the portion that needs a lock is less than 1.5% of
> execution time and so there's plenty to still run in parallel:
> ...
>       - 32.59% __perf_event__synthesize_threads
>          - 32.54% __event__synthesize_thread
>             + 22.13% perf_event__synthesize_mmap_events
>             + 6.68% perf_event__get_comm_ids.constprop.0
>             + 1.49% process_synthesized_event
>             + 1.29% __GI___readdir64
>             + 0.60% __opendir
> ...
> 
> The multi-threaded benchmark in this patch (pass -t):
> https://lore.kernel.org/lkml/20200415054050.31645-2-irogers@google.com/
> shows:
> 
> Computing performance of multi threaded perf event synthesis by
> synthesizing events on CPU 0:
>  Number of synthesis threads: 1
>    Average synthesis took: 127729.000 usec (+- 3372.880 usec)
>    Average num. events: 21548.600 (+- 0.306)
>    Average time per event 5.927 usec
>  Number of synthesis threads: 2
>    Average synthesis took: 88863.500 usec (+- 385.168 usec)
>    Average num. events: 21552.800 (+- 0.327)
>    Average time per event 4.123 usec
>  Number of synthesis threads: 3
>    Average synthesis took: 83257.400 usec (+- 348.617 usec)
>    Average num. events: 21553.200 (+- 0.327)
>    Average time per event 3.863 usec
>  Number of synthesis threads: 4
>    Average synthesis took: 75093.000 usec (+- 422.978 usec)
>    Average num. events: 21554.200 (+- 0.200)
>    Average time per event 3.484 usec
>  Number of synthesis threads: 5
>    Average synthesis took: 64896.600 usec (+- 353.348 usec)
>    Average num. events: 21558.000 (+- 0.000)
>    Average time per event 3.010 usec
>  Number of synthesis threads: 6
>    Average synthesis took: 59210.200 usec (+- 342.890 usec)
>    Average num. events: 21560.000 (+- 0.000)
>    Average time per event 2.746 usec
>  Number of synthesis threads: 7
>    Average synthesis took: 54093.900 usec (+- 306.247 usec)
>    Average num. events: 21562.000 (+- 0.000)
>    Average time per event 2.509 usec
>  Number of synthesis threads: 8
>    Average synthesis took: 48938.700 usec (+- 341.732 usec)
>    Average num. events: 21564.000 (+- 0.000)
>    Average time per event 2.269 usec
> 
> The event logic there is using an atomic rather than a lock and the
> scaling isn't linear as not all the logic is threaded. Still with 8
> threads we see things going about 2.6 times faster. On a large loaded
> machine that may bring 10 seconds of event synthesis down to less than
> 4. On a desktop there's no measurable difference and the
> --num-thread-synthesize is defaulted to 1.

ok, nice ;) sorry I did not get to this before you posted v2,
but could you plz send v3 with above in the changelog?

thanks,
jirka

Re: [PATCH] perf/record: add num-synthesize-threads option

[Ian Rogers]

On Wed, Apr 22, 2020 at 1:15 AM Jiri Olsa <jolsa@redhat.com> wrote:
>
> On Mon, Apr 20, 2020 at 05:31:41PM -0700, Ian Rogers wrote:
>
> SNIP
>
> > > > +{
> > > > +     static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER;
> > > > +     int ret;
> > > > +
> > > > +     pthread_mutex_lock(&synth_lock);
> > > > +     ret = process_synthesized_event(tool, event, sample, machine);
> > > > +     pthread_mutex_unlock(&synth_lock);
> > >
> > > hum, so how much faster is the synthesizing with threads in record,
> > > given that we serialize it on every event that goes to the file?
> >
> > We see long synthesis times of the order seconds on loaded >100 core
> > servers. I've not been able to create a reproduction on my desktop.
> > You are right that making synthesis multithreaded will suffer from
> > Amdahl's law if the write is a synchronization point. Measuring with
> > the following patch in place:
> > https://lore.kernel.org/lkml/20200415054050.31645-4-irogers@google.com/
> > without threads the portion that needs a lock is less than 1.5% of
> > execution time and so there's plenty to still run in parallel:
> > ...
> >       - 32.59% __perf_event__synthesize_threads
> >          - 32.54% __event__synthesize_thread
> >             + 22.13% perf_event__synthesize_mmap_events
> >             + 6.68% perf_event__get_comm_ids.constprop.0
> >             + 1.49% process_synthesized_event
> >             + 1.29% __GI___readdir64
> >             + 0.60% __opendir
> > ...
> >
> > The multi-threaded benchmark in this patch (pass -t):
> > https://lore.kernel.org/lkml/20200415054050.31645-2-irogers@google.com/
> > shows:
> >
> > Computing performance of multi threaded perf event synthesis by
> > synthesizing events on CPU 0:
> >  Number of synthesis threads: 1
> >    Average synthesis took: 127729.000 usec (+- 3372.880 usec)
> >    Average num. events: 21548.600 (+- 0.306)
> >    Average time per event 5.927 usec
> >  Number of synthesis threads: 2
> >    Average synthesis took: 88863.500 usec (+- 385.168 usec)
> >    Average num. events: 21552.800 (+- 0.327)
> >    Average time per event 4.123 usec
> >  Number of synthesis threads: 3
> >    Average synthesis took: 83257.400 usec (+- 348.617 usec)
> >    Average num. events: 21553.200 (+- 0.327)
> >    Average time per event 3.863 usec
> >  Number of synthesis threads: 4
> >    Average synthesis took: 75093.000 usec (+- 422.978 usec)
> >    Average num. events: 21554.200 (+- 0.200)
> >    Average time per event 3.484 usec
> >  Number of synthesis threads: 5
> >    Average synthesis took: 64896.600 usec (+- 353.348 usec)
> >    Average num. events: 21558.000 (+- 0.000)
> >    Average time per event 3.010 usec
> >  Number of synthesis threads: 6
> >    Average synthesis took: 59210.200 usec (+- 342.890 usec)
> >    Average num. events: 21560.000 (+- 0.000)
> >    Average time per event 2.746 usec
> >  Number of synthesis threads: 7
> >    Average synthesis took: 54093.900 usec (+- 306.247 usec)
> >    Average num. events: 21562.000 (+- 0.000)
> >    Average time per event 2.509 usec
> >  Number of synthesis threads: 8
> >    Average synthesis took: 48938.700 usec (+- 341.732 usec)
> >    Average num. events: 21564.000 (+- 0.000)
> >    Average time per event 2.269 usec
> >
> > The event logic there is using an atomic rather than a lock and the
> > scaling isn't linear as not all the logic is threaded. Still with 8
> > threads we see things going about 2.6 times faster. On a large loaded
> > machine that may bring 10 seconds of event synthesis down to less than
> > 4. On a desktop there's no measurable difference and the
> > --num-thread-synthesize is defaulted to 1.
>
> ok, nice ;) sorry I did not get to this before you posted v2,
> but could you plz send v3 with above in the changelog?

Done.
https://lore.kernel.org/lkml/20200422155038.9380-1-irogers@google.com/T/#u
Not done initially as I wasn't sure the argument was coherent and also
the other two patches haven't landed - it would be nice to avoid
linking to the patches on the mailing list.

Thanks,
Ian

> thanks,
> jirka
>