[PATCH 0/2] perf bpf: Add helper to support generate BPF object

[PATCH 0/2] perf bpf: Add helper to support generate BPF object

[Wang Nan]

These 2 patch introduce a new perfconfig option (llvm.dump-obj) and a
new cmdline option (perf record --dry-run). Combined together, perf user
is able to generate a BPF object for an embedded platform on a x86
server using perf, instead directly using llvm cmdline and provide -I
options and default macros.

Wang Nan (2):
  perf llvm: Allow dump llvm output object file using llvm.dump-obj
  perf record: Add --dry-run option to check cmdline options

 tools/perf/Documentation/perf-record.txt |  7 ++++++
 tools/perf/builtin-record.c              |  7 ++++++
 tools/perf/util/llvm-utils.c             | 42 ++++++++++++++++++++++++++++++++
 tools/perf/util/llvm-utils.h             |  5 ++++
 4 files changed, 61 insertions(+)

-- 
1.8.3.4

[PATCH 1/2] perf llvm: Allow dump llvm output object file using llvm.dump-obj

[Wang Nan]

Add a 'llvm.dump-obj' config option to enable perf dump BPF object files
compiled by LLVM.

This option is useful when using BPF objects in embedded platforms.
LLVM compiler won't be deployed in these platforms, and currently we
don't support dynamic compiling library.

Before this patch users have to explicitly issue llvm commands to
compile BPF scripts, and can't use helpers (like include path detection
and default macros) in perf. With this option, user is allowed to use
perf to compile their BPF objects then copy them into their embedded
platforms.

Signed-off-by: Wang Nan <wangnan0@huawei.com>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Jiri Olsa <jolsa@kernel.org>
---
 tools/perf/util/llvm-utils.c | 42 ++++++++++++++++++++++++++++++++++++++++++
 tools/perf/util/llvm-utils.h |  5 +++++
 2 files changed, 47 insertions(+)

diff --git a/tools/perf/util/llvm-utils.c b/tools/perf/util/llvm-utils.c
index 33071d6..ede8f7d 100644
--- a/tools/perf/util/llvm-utils.c
+++ b/tools/perf/util/llvm-utils.c
@@ -42,6 +42,8 @@ int perf_llvm_config(const char *var, const char *value)
 		llvm_param.kbuild_dir = strdup(value);
 	else if (!strcmp(var, "kbuild-opts"))
 		llvm_param.kbuild_opts = strdup(value);
+	else if (!strcmp(var, "dump-obj"))
+		llvm_param.dump_obj = !!perf_config_bool(var, value);
 	else
 		return -1;
 	llvm_param.user_set_param = true;
@@ -326,6 +328,42 @@ get_kbuild_opts(char **kbuild_dir, char **kbuild_include_opts)
 	pr_debug("include option is set to %s\n", *kbuild_include_opts);
 }
 
+static void
+dump_obj(const char *path, void *obj_buf, size_t size)
+{
+	char *obj_path = strdup(path);
+	FILE *fp;
+	char *p;
+
+	if (!obj_path) {
+		pr_warning("WARNING: No enough memory, skip object dumpping\n");
+		return;
+	}
+
+	p = strrchr(obj_path, '.');
+	if (!p || (strcmp(p, ".c") != 0)) {
+		pr_warning("WARNING: invalid llvm source path: '%s', skip object dumpping\n",
+			   obj_path);
+		goto out;
+	}
+
+	p[1] = 'o';
+	fp = fopen(obj_path, "wb");
+	if (!fp) {
+		pr_warning("WARNING: failed to open '%s': %s, skip object dumpping\n",
+			   obj_path, strerror(errno));
+		goto out;
+	}
+
+	pr_info("LLVM: dumpping %s\n", obj_path);
+	if (fwrite(obj_buf, size, 1, fp) != 1)
+		pr_warning("WARNING: failed to write to file '%s': %s, skip object dumpping\n",
+			   obj_path, strerror(errno));
+	fclose(fp);
+out:
+	free(obj_path);
+}
+
 int llvm__compile_bpf(const char *path, void **p_obj_buf,
 		      size_t *p_obj_buf_sz)
 {
@@ -411,6 +449,10 @@ int llvm__compile_bpf(const char *path, void **p_obj_buf,
 
 	free(kbuild_dir);
 	free(kbuild_include_opts);
+
+	if (llvm_param.dump_obj)
+		dump_obj(path, obj_buf, obj_buf_sz);
+
 	if (!p_obj_buf)
 		free(obj_buf);
 	else
diff --git a/tools/perf/util/llvm-utils.h b/tools/perf/util/llvm-utils.h
index 23b9a74..9f501ce 100644
--- a/tools/perf/util/llvm-utils.h
+++ b/tools/perf/util/llvm-utils.h
@@ -30,6 +30,11 @@ struct llvm_param {
 	 */
 	const char *kbuild_opts;
 	/*
+	 * Default is false. If set to true, write compiling result
+	 * to object file.
+	 */
+	bool dump_obj;
+	/*
 	 * Default is false. If one of the above fields is set by user
 	 * explicitly then user_set_llvm is set to true. This is used
 	 * for perf test. If user doesn't set anything in .perfconfig
-- 
1.8.3.4

[PATCH 2/2] perf record: Add --dry-run option to check cmdline options

[Wang Nan]

With '--dry-run', 'perf record' doesn't do reall recording. Combine with
llvm.dump-obj option, --dry-run can be used to help compile BPF objects for
embedded platform.

Signed-off-by: Wang Nan <wangnan0@huawei.com>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Jiri Olsa <jolsa@kernel.org>
---
 tools/perf/Documentation/perf-record.txt | 7 +++++++
 tools/perf/builtin-record.c              | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt
index 8dbee83..5b46b1d 100644
--- a/tools/perf/Documentation/perf-record.txt
+++ b/tools/perf/Documentation/perf-record.txt
@@ -360,6 +360,13 @@ particular perf.data snapshot should be kept or not.
 
 Implies --timestamp-filename, --no-buildid and --no-buildid-cache.
 
+--dry-run::
+Parse options then exit. --dry-run can be used to detect errors in cmdline
+options.
+
+'perf record --dry-run -e' can act as a BPF script compiler if llvm.dump-obj
+in config file is set to true.
+
 SEE ALSO
 --------
 linkperf:perf-stat[1], linkperf:perf-list[1]
diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
index d4cf1b0..b1304eb 100644
--- a/tools/perf/builtin-record.c
+++ b/tools/perf/builtin-record.c
@@ -1274,6 +1274,8 @@ static struct record record = {
 const char record_callchain_help[] = CALLCHAIN_RECORD_HELP
 	"\n\t\t\t\tDefault: fp";
 
+static bool dry_run;
+
 /*
  * XXX Will stay a global variable till we fix builtin-script.c to stop messing
  * with it and switch to use the library functions in perf_evlist that came
@@ -1393,6 +1395,8 @@ struct option __record_options[] = {
 		    "append timestamp to output filename"),
 	OPT_BOOLEAN(0, "switch-output", &record.switch_output,
 		    "Switch output when receive SIGUSR2"),
+	OPT_BOOLEAN(0, "dry-run", &dry_run,
+		    "Parse options then exit"),
 	OPT_END()
 };
 
@@ -1462,6 +1466,9 @@ int cmd_record(int argc, const char **argv, const char *prefix __maybe_unused)
 	if (err)
 		return err;
 
+	if (dry_run)
+		return 0;
+
 	err = bpf__setup_stdout(rec->evlist);
 	if (err) {
 		bpf__strerror_setup_stdout(rec->evlist, err, errbuf, sizeof(errbuf));
-- 
1.8.3.4

Re: [PATCH 2/2] perf record: Add --dry-run option to check cmdline options

[Arnaldo Carvalho de Melo]

Em Thu, Jun 16, 2016 at 08:02:41AM +0000, Wang Nan escreveu:
> With '--dry-run', 'perf record' doesn't do reall recording. Combine with
> llvm.dump-obj option, --dry-run can be used to help compile BPF objects for
> embedded platform.

So these are nice and have value, but can we have a subcommand to do all
this with an expressive name, Something like:

  perf bpfcc foo.c -o foo

or shorter:

  perf bcc foo.c -o foo

Just like one would use gcc or some other compiler to generate something
for later use?

That if called as:

  perf bcc foo.c

Would default to generating a foo.o file.

  Then, later, one could use this as a event name, i.e.

  trace --event foo

Would, knowing that there is no event named "foo", look at the current
directory (and in some other places perhaps) for a file named "foo" that
was a bpf object file to use as it would a foo.c, shortcircuiting the
bpf compilation code.

If this was done instead:

  trace --event foo.c

And foo.c wasn't present, it would fallback to the behaviour described
in the previous paragraph: look for a foo.o or foo bpf object file, etc.

What do you think?

- Arnaldo
 
> Signed-off-by: Wang Nan <wangnan0@huawei.com>
> Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
> Cc: Alexei Starovoitov <ast@kernel.org>
> Cc: Jiri Olsa <jolsa@kernel.org>
> ---
>  tools/perf/Documentation/perf-record.txt | 7 +++++++
>  tools/perf/builtin-record.c              | 7 +++++++
>  2 files changed, 14 insertions(+)
> 
> diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt
> index 8dbee83..5b46b1d 100644
> --- a/tools/perf/Documentation/perf-record.txt
> +++ b/tools/perf/Documentation/perf-record.txt
> @@ -360,6 +360,13 @@ particular perf.data snapshot should be kept or not.
>  
>  Implies --timestamp-filename, --no-buildid and --no-buildid-cache.
>  
> +--dry-run::
> +Parse options then exit. --dry-run can be used to detect errors in cmdline
> +options.
> +
> +'perf record --dry-run -e' can act as a BPF script compiler if llvm.dump-obj
> +in config file is set to true.
> +
>  SEE ALSO
>  --------
>  linkperf:perf-stat[1], linkperf:perf-list[1]
> diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
> index d4cf1b0..b1304eb 100644
> --- a/tools/perf/builtin-record.c
> +++ b/tools/perf/builtin-record.c
> @@ -1274,6 +1274,8 @@ static struct record record = {
>  const char record_callchain_help[] = CALLCHAIN_RECORD_HELP
>  	"\n\t\t\t\tDefault: fp";
>  
> +static bool dry_run;
> +
>  /*
>   * XXX Will stay a global variable till we fix builtin-script.c to stop messing
>   * with it and switch to use the library functions in perf_evlist that came
> @@ -1393,6 +1395,8 @@ struct option __record_options[] = {
>  		    "append timestamp to output filename"),
>  	OPT_BOOLEAN(0, "switch-output", &record.switch_output,
>  		    "Switch output when receive SIGUSR2"),
> +	OPT_BOOLEAN(0, "dry-run", &dry_run,
> +		    "Parse options then exit"),
>  	OPT_END()
>  };
>  
> @@ -1462,6 +1466,9 @@ int cmd_record(int argc, const char **argv, const char *prefix __maybe_unused)
>  	if (err)
>  		return err;
>  
> +	if (dry_run)
> +		return 0;
> +
>  	err = bpf__setup_stdout(rec->evlist);
>  	if (err) {
>  		bpf__strerror_setup_stdout(rec->evlist, err, errbuf, sizeof(errbuf));
> -- 
> 1.8.3.4

Re: [PATCH 2/2] perf record: Add --dry-run option to check cmdline options

[Arnaldo Carvalho de Melo]

Em Thu, Jun 16, 2016 at 01:48:15PM -0300, Arnaldo Carvalho de Melo escreveu:
> Em Thu, Jun 16, 2016 at 08:02:41AM +0000, Wang Nan escreveu:
> > With '--dry-run', 'perf record' doesn't do reall recording. Combine with
> > llvm.dump-obj option, --dry-run can be used to help compile BPF objects for
> > embedded platform.
> 
> So these are nice and have value, but can we have a subcommand to do all
> this with an expressive name, Something like:
> 
>   perf bpfcc foo.c -o foo
> 
> or shorter:
> 
>   perf bcc foo.c -o foo

BTW, I just applied both files. Testing them now.

- Arnaldo

Re: [PATCH 2/2] perf record: Add --dry-run option to check cmdline options

[Wangnan (F)]

On 2016/6/17 0:48, Arnaldo Carvalho de Melo wrote:
> Em Thu, Jun 16, 2016 at 08:02:41AM +0000, Wang Nan escreveu:
>> With '--dry-run', 'perf record' doesn't do reall recording. Combine with
>> llvm.dump-obj option, --dry-run can be used to help compile BPF objects for
>> embedded platform.
> So these are nice and have value, but can we have a subcommand to do all
> this with an expressive name, Something like:
>
>    perf bpfcc foo.c -o foo
>
> or shorter:
>
>    perf bcc foo.c -o foo
>
> Just like one would use gcc or some other compiler to generate something
> for later use?

I'll try it today. I thought a subcommand require a bigger feature,
and wrapping clang is not big enough.

> That if called as:
>
>    perf bcc foo.c
>
> Would default to generating a foo.o file.
>
>    Then, later, one could use this as a event name, i.e.
>
>    trace --event foo
>
> Would, knowing that there is no event named "foo", look at the current
> directory (and in some other places perhaps) for a file named "foo" that
> was a bpf object file to use as it would a foo.c, shortcircuiting the
> bpf compilation code.
> If this was done instead:
>
>    trace --event foo.c
>
> And foo.c wasn't present, it would fallback to the behaviour described
> in the previous paragraph: look for a foo.o or foo bpf object file, etc.
>
> What do you think?

I'm not sure how many people can be benified from this feature. The only
advantage I can understand is we can skip the '.c', '.o' or '.bpf' suffix.

I guess what you really want is introducing something like buildid-cache for
BPF object. One can compile his/her BPF scriptlets into .o using 'perf 
bcc' and
insert it into cache, then he/her can use the resuling object without 
remembering
the path of it.

About fallback, if user explicitly uses '.o' or '.bpf' as suffix our 
parser can
be easier. Technically we need a boundary to split event name and 
configuration.
'.c', '.o' and '.bpf' are boundaries. In addition, is there any 
difference between
'-e mybpf' and '-e mybpf.bpf'? We can define that, when using '-e mybpf'
the search path whould be the BPF object cache, when using '-e 
mybpf.bpf' the
search path is current directory. It is acceptable, but why not make '-e 
mybpf.bpf'
search BPF object cache also?

Thank you.

Re: [PATCH 2/2] perf record: Add --dry-run option to check cmdline options

[Arnaldo Carvalho de Melo]

Em Mon, Jun 20, 2016 at 11:29:13AM +0800, Wangnan (F) escreveu:
> On 2016/6/17 0:48, Arnaldo Carvalho de Melo wrote:
> >Em Thu, Jun 16, 2016 at 08:02:41AM +0000, Wang Nan escreveu:
> >>With '--dry-run', 'perf record' doesn't do reall recording. Combine with
> >>llvm.dump-obj option, --dry-run can be used to help compile BPF objects for
> >>embedded platform.
> >So these are nice and have value, but can we have a subcommand to do all
> >this with an expressive name, Something like:

> >   perf bpfcc foo.c -o foo

> >or shorter:

> >   perf bcc foo.c -o foo

> >Just like one would use gcc or some other compiler to generate something
> >for later use?

> I'll try it today. I thought a subcommand require a bigger feature,
> and wrapping clang is not big enough.

Not really, we may have as many as we like, given that they provide
something useful, like I think is the case here.

Having to edit ~/.perfconfig, create a new section, a variable in it
with a boolean value (at first, just reading the changeset comment, I
thought I had to provide a directory where to store the objects
"dumped"), to then use a tool to record a .c event, but not recording
(use dry-run, which is useful to test the command line, etc), to then
get, on the current directory, the end result looked to me a convoluted
way to ask perf to compile the given .c file into a .o for later use.

Doing:

	perf bcc -c foo.c

Looks so much simpler and similar to an existing compile source code
into object file workflow (gcc's, any C compiler) that I think it would
fit in the workflow being discussed really nicely.

> >That if called as:
> >
> >   perf bcc foo.c
> >
> >Would default to generating a foo.o file.
> >
> >   Then, later, one could use this as a event name, i.e.
> >
> >   trace --event foo
> >
> >Would, knowing that there is no event named "foo", look at the current
> >directory (and in some other places perhaps) for a file named "foo" that
> >was a bpf object file to use as it would a foo.c, shortcircuiting the
> >bpf compilation code.
> >If this was done instead:
> >
> >   trace --event foo.c
> >
> >And foo.c wasn't present, it would fallback to the behaviour described
> >in the previous paragraph: look for a foo.o or foo bpf object file, etc.
> >
> >What do you think?
> 
> I'm not sure how many people can be benified from this feature. The only
> advantage I can understand is we can skip the '.c', '.o' or '.bpf' suffix.
> 
> I guess what you really want is introducing something like buildid-cache for
> BPF object. One can compile his/her BPF scriptlets into .o using

Nope, the build id cache is that, a cache, somewhere to store object
files that had samples taken in some previous tool session for later
use.

Sure, we can store bpf .o files there, keyed by its build-id, etc, and
then store in the perf.data file the build-id to get it from the cache,
so that we could re-run that workload without having to go thru the
process of recompiling the .c bpf file, if that can be done (running it
on the same kernel, perhaps on a different machine, etc).

> 'perf bcc' and insert it into cache, then he/her can use the resuling
> object without remembering the path of it.

Well, this looks similar to what we do when we try to find a vmlinux
file, we look at a vmlinux_path, searching for a suitable file that has
the matching build-id, i.e. look at the current directory, then at
/boot/, /lib/modules/`unamr -r`, /usr/lib/debug, build-id cache, etc.

The key here is to be able to register the .o file used in the perf.data
file without copying it, i.e. storing just its build-id in the perf.data
file build-id table.
 
> About fallback, if user explicitly uses '.o' or '.bpf' as suffix our
> parser can be easier. Technically we need a boundary to split event
> name and configuration.  '.c', '.o' and '.bpf' are boundaries. In
> addition, is there any difference between '-e mybpf' and '-e
> mybpf.bpf'? We can define that, when using '-e mybpf' the search path
> whould be the BPF object cache, when using '-e mybpf.bpf' the search
> path is current directory. It is acceptable, but why not make '-e
> mybpf.bpf' search BPF object cache also?

Well there is a namespace issue here, if we say:

	perf record -e cycles

then this is well known, we want PERF_TYPE_HARDWARE,
PERF_COUNT_HW_CPU_CYCLES. If we instead use:

	perf record -e cycles.c

Then this also is well known, we need to build this somehow, and right
now the only way to do this is to use the llvm/clang infrastructure and
then load it into the kernel via sys_bpf.

If we say:

	perf record -e cycles.bpf

Then we don't have anything associated with this and may go on trying to
map it to a PERF_TYPE_HARDWARE, PERF_TYPE_SOFTWARE, etc till we find a
suitable event, i.e. if it doesn't match anything, we would end up
looking at a file in the current directory, figure out it is an ELF file
and that its contents are a BPF proggie, that we would load via sys_bpf,
etc.

But what I was proposing was to stick to what we have now, i.e. 

	perf record -e cycles.c

Means build and load an eBPF proggie via the clang/llvm infrastructure
and sys_bpf().

But... before doing that, look at the current directory (and the BPF proggie
path, that would include the build-id cache, like we do to find a vmlinux) to
find an object file that matches the cycles.c contents, to avoid having to run
clang/llvm everytime we specify that .c eBPF event, and ultimately to remove
the requirement that we have the clang/llvm tools installed.

I.e. we would calculate a build-id from the .c file contents and then
look at the bpf pre-built proggie object path.

For binaries we can have an ELF section in an object file where we store
the build-id to avoid having to calculate it everytime we need it, for
.c files on filesystems with extended attributes we could use
"user.checksum.sha256" as our build-id :-)

E.g.:

[acme@jouet ~]$ echo Hello, world > hello
[acme@jouet ~]$ setfattr -n user.checksum.sha256 -v `sha256sum hello | cut -d' ' -f1` hello 
[acme@jouet ~]$ getfattr -n user.checksum.sha256 hello 
# file: hello
user.checksum.sha256="37980c33951de6b0e450c3701b219bfeee930544705f637cd1158b63827bb390"

[acme@jouet ~]$ cat hello
Hello, world
[acme@jouet ~]$ sha256sum hello 
37980c33951de6b0e450c3701b219bfeee930544705f637cd1158b63827bb390  hello
[acme@jouet ~]$

Having a .build-id section in the .o bpf file would be nice for that :-)

Note, "hello" above would be our .c bpf file, and that sha256sum would be our build-id that
would allow us to find the right .o file to use for a give .c file, in an environment
_without_ llvm/clang.

If it can't be found, perf would say that no suitable .o file matching that .c file
was found, go build one in your developer machine and then copy it over to the
machine where you want to use it without a llvm/clang environment.

- Arnaldo

Re: [PATCH 2/2] perf record: Add --dry-run option to check cmdline options

[Alexei Starovoitov]

On Mon, Jun 20, 2016 at 11:38:18AM -0300, Arnaldo Carvalho de Melo wrote:
> Em Mon, Jun 20, 2016 at 11:29:13AM +0800, Wangnan (F) escreveu:
> > On 2016/6/17 0:48, Arnaldo Carvalho de Melo wrote:
> > >Em Thu, Jun 16, 2016 at 08:02:41AM +0000, Wang Nan escreveu:
> > >>With '--dry-run', 'perf record' doesn't do reall recording. Combine with
> > >>llvm.dump-obj option, --dry-run can be used to help compile BPF objects for
> > >>embedded platform.
> > >So these are nice and have value, but can we have a subcommand to do all
> > >this with an expressive name, Something like:
> 
> > >   perf bpfcc foo.c -o foo
> 
> > >or shorter:
> 
> > >   perf bcc foo.c -o foo
> 
> > >Just like one would use gcc or some other compiler to generate something
> > >for later use?
> 
> > I'll try it today. I thought a subcommand require a bigger feature,
> > and wrapping clang is not big enough.
> 
> Not really, we may have as many as we like, given that they provide
> something useful, like I think is the case here.
> 
> Having to edit ~/.perfconfig, create a new section, a variable in it
> with a boolean value (at first, just reading the changeset comment, I
> thought I had to provide a directory where to store the objects
> "dumped"), to then use a tool to record a .c event, but not recording
> (use dry-run, which is useful to test the command line, etc), to then
> get, on the current directory, the end result looked to me a convoluted
> way to ask perf to compile the given .c file into a .o for later use.
> 
> Doing:
> 
> 	perf bcc -c foo.c
> 
> Looks so much simpler and similar to an existing compile source code
> into object file workflow (gcc's, any C compiler) that I think it would
> fit in the workflow being discussed really nicely.

I'm hopeful that eventually we'll be able merge iovisor/bcc project
with perf, so would be good to reserve 'perf bcc' command for that
future use. Also picking a different name for compiling would be less
confusing to users who already familiar with bcc. Instead we can use:
perf bpfcc foo.c -o foo.o
perf cc foo.c
perf compile foo.c

Re: [PATCH 2/2] perf record: Add --dry-run option to check cmdline options

[Arnaldo Carvalho de Melo]

Em Mon, Jun 20, 2016 at 09:22:11AM -0700, Alexei Starovoitov escreveu:
> On Mon, Jun 20, 2016 at 11:38:18AM -0300, Arnaldo Carvalho de Melo wrote:
> > Doing:

> > 	perf bcc -c foo.c

> > Looks so much simpler and similar to an existing compile source code
> > into object file workflow (gcc's, any C compiler) that I think it would
> > fit in the workflow being discussed really nicely.
 
> I'm hopeful that eventually we'll be able merge iovisor/bcc project
> with perf, so would be good to reserve 'perf bcc' command for that
> future use. Also picking a different name for compiling would be less
> confusing to users who already familiar with bcc. Instead we can use:
> perf bpfcc foo.c -o foo.o
> perf cc foo.c

'perf cc' seems sensible, and has the added bonus of being one letter
shorter :-)

- Arnaldo

> perf compile foo.c

Re: [PATCH 2/2] perf record: Add --dry-run option to check cmdline options

[David Ahern]

On 6/20/16 12:13 PM, Arnaldo Carvalho de Melo wrote:
> 'perf cc' seems sensible, and has the added bonus of being one letter
> shorter :-)


perf is now a general front-end to a compiler?

Re: [PATCH 2/2] perf record: Add --dry-run option to check cmdline options

[Arnaldo Carvalho de Melo]

Em Mon, Jun 20, 2016 at 12:16:55PM -0600, David Ahern escreveu:
> On 6/20/16 12:13 PM, Arnaldo Carvalho de Melo wrote:
> > 'perf cc' seems sensible, and has the added bonus of being one letter
> > shorter :-)
 
> perf is now a general front-end to a compiler?

Well, it is for quite a while already, what we're talking about here is
to have this:

  # cat filter.c 
  #include <uapi/linux/bpf.h>
  #define SEC(NAME) __attribute__((section(NAME), used))

  SEC("func=hrtimer_nanosleep rqtp->tv_nsec")
  int func(void *ctx, int err, long nsec)
  {
	return nsec > 1000;
  }
  char _license[] SEC("license") = "GPL";
  int _version SEC("version") = LINUX_VERSION_CODE;
  # perf trace -e nanosleep --event filter.c usleep 1
     0.063 ( 0.063 ms): usleep/8041 nanosleep(rqtp: 0x7fff62bead80) = 0
  # perf trace -e nanosleep --event filter.c usleep 2
     0.008 ( 0.008 ms): usleep/8325 nanosleep(rqtp: 0x7ffc2afdf3b0) ...
     0.008 (         ): perf_bpf_probe:func:(ffffffff811137d0) tv_nsec=2000)
     0.070 ( 0.070 ms): usleep/8325  ... [continued]: nanosleep()) = 0
  # 

To not cal the clang compiler under the hood all the time, i.e.
pre-building the .o file that will then be used when present.

What Wang did was to make that possible by adding this to ~/.perfconfig:

  # cat ~/.perfconfig 
  [llvm]
	dump-obj = true
  # 

This way, when we run we get:

  # trace -e nanosleep --event filter.c usleep 6
  LLVM: dumpping filter.o
     0.008 ( 0.008 ms): usleep/9189 nanosleep(rqtp: 0x7fff97a704d0                                        ) ...
     0.008 (         ): perf_bpf_probe:func:(ffffffff811137d0) tv_nsec=6000)
     0.070 ( 0.070 ms): usleep/9189  ... [continued]: nanosleep()) = 0
  #
  # file filter.o
  filter.o: ELF 64-bit LSB relocatable, no machine, version 1 (SYSV), not stripped
  # readelf -SW filter.o
  There are 7 section headers, starting at offset 0x148:

  Section Headers:
    [Nr] Name              Type            Address          Off    Size   ES Flg Lk Inf Al
    [ 0]                   NULL            0000000000000000 000000 000000 00      0   0  0
    [ 1] .strtab           STRTAB          0000000000000000 0000e8 00005a 00      0   0  1
    [ 2] .text             PROGBITS        0000000000000000 000040 000000 00  AX  0   0  4
    [ 3] func=hrtimer_nanosleep rqtp->tv_nsec PROGBITS        0000000000000000 000040 000028 00  AX  0   0  8
    [ 4] license           PROGBITS        0000000000000000 000068 000004 00  WA  0   0  1
    [ 5] version           PROGBITS        0000000000000000 00006c 000004 00  WA  0   0  4
    [ 6] .symtab           SYMTAB          0000000000000000 000070 000078 18      1   2  8
  Key to Flags:
    W (write), A (alloc), X (execute), M (merge), S (strings)
    I (info), L (link order), G (group), T (TLS), E (exclude), x (unknown)
    O (extra OS processing required) o (OS specific), p (processor specific)
  #

Generating this .o file explicitely and then, when found and somehow checked
that it matches what is in filter.c, shortcircuit the process bypassing the
clang call and using filter.o directly.

This will remove the need for having clang in embedded systems, for instance,
and will speed up using eBPF scripts with perf.

- Arnaldo

Re: [PATCH 2/2] perf record: Add --dry-run option to check cmdline options

[Wangnan (F)]

On 2016/6/20 22:38, Arnaldo Carvalho de Melo wrote:
> Em Mon, Jun 20, 2016 at 11:29:13AM +0800, Wangnan (F) escreveu:
>> On 2016/6/17 0:48, Arnaldo Carvalho de Melo wrote:
>>> Em Thu, Jun 16, 2016 at 08:02:41AM +0000, Wang Nan escreveu:

[SNIP]

>> About fallback, if user explicitly uses '.o' or '.bpf' as suffix our
>> parser can be easier. Technically we need a boundary to split event
>> name and configuration.  '.c', '.o' and '.bpf' are boundaries. In
>> addition, is there any difference between '-e mybpf' and '-e
>> mybpf.bpf'? We can define that, when using '-e mybpf' the search path
>> whould be the BPF object cache, when using '-e mybpf.bpf' the search
>> path is current directory. It is acceptable, but why not make '-e
>> mybpf.bpf' search BPF object cache also?
> Well there is a namespace issue here, if we say:
>
> 	perf record -e cycles
>
> then this is well known, we want PERF_TYPE_HARDWARE,
> PERF_COUNT_HW_CPU_CYCLES. If we instead use:
>
> 	perf record -e cycles.c
>
> Then this also is well known, we need to build this somehow, and right
> now the only way to do this is to use the llvm/clang infrastructure and
> then load it into the kernel via sys_bpf.
>
> If we say:
>
> 	perf record -e cycles.bpf
>
> Then we don't have anything associated with this and may go on trying to
> map it to a PERF_TYPE_HARDWARE, PERF_TYPE_SOFTWARE, etc till we find a
> suitable event, i.e. if it doesn't match anything, we would end up
> looking at a file in the current directory, figure out it is an ELF file
> and that its contents are a BPF proggie, that we would load via sys_bpf,
> etc.

cycles.bpf is not a good example. See tools/perf/util/parse-events.l:
   ...
   bpf_object      .*\.(o|bpf)
   ...

currently '.o' equals to '.bpf'.

Thank you.

Re: [PATCH 2/2] perf record: Add --dry-run option to check cmdline options

[Namhyung Kim]

On Tue, Jun 21, 2016 at 3:13 AM, Arnaldo Carvalho de Melo
<acme@kernel.org> wrote:
> Em Mon, Jun 20, 2016 at 09:22:11AM -0700, Alexei Starovoitov escreveu:
>> On Mon, Jun 20, 2016 at 11:38:18AM -0300, Arnaldo Carvalho de Melo wrote:
>> > Doing:
>
>> >     perf bcc -c foo.c
>
>> > Looks so much simpler and similar to an existing compile source code
>> > into object file workflow (gcc's, any C compiler) that I think it would
>> > fit in the workflow being discussed really nicely.
>
>> I'm hopeful that eventually we'll be able merge iovisor/bcc project
>> with perf, so would be good to reserve 'perf bcc' command for that
>> future use. Also picking a different name for compiling would be less
>> confusing to users who already familiar with bcc. Instead we can use:
>> perf bpfcc foo.c -o foo.o
>> perf cc foo.c
>
> 'perf cc' seems sensible, and has the added bonus of being one letter
> shorter :-)
>
> - Arnaldo
>
>> perf compile foo.c

What about this?

perf bpf --compile foo.c   or,
perf bpf --cc foo.c

Thanks,
Namhyung

Re: [PATCH 2/2] perf record: Add --dry-run option to check cmdline options

[David Ahern]

On 6/20/16 8:02 PM, Namhyung Kim wrote:
> On Tue, Jun 21, 2016 at 3:13 AM, Arnaldo Carvalho de Melo
> <acme@kernel.org> wrote:
>> Em Mon, Jun 20, 2016 at 09:22:11AM -0700, Alexei Starovoitov escreveu:
>>> On Mon, Jun 20, 2016 at 11:38:18AM -0300, Arnaldo Carvalho de Melo wrote:
>>>> Doing:
>>
>>>>     perf bcc -c foo.c
>>
>>>> Looks so much simpler and similar to an existing compile source code
>>>> into object file workflow (gcc's, any C compiler) that I think it would
>>>> fit in the workflow being discussed really nicely.
>>
>>> I'm hopeful that eventually we'll be able merge iovisor/bcc project
>>> with perf, so would be good to reserve 'perf bcc' command for that
>>> future use. Also picking a different name for compiling would be less
>>> confusing to users who already familiar with bcc. Instead we can use:
>>> perf bpfcc foo.c -o foo.o
>>> perf cc foo.c
>>
>> 'perf cc' seems sensible, and has the added bonus of being one letter
>> shorter :-)
>>
>> - Arnaldo
>>
>>> perf compile foo.c
>
> What about this?
>
> perf bpf --compile foo.c   or,
> perf bpf --cc foo.c

That sounds more reasonable to me than 'perf cc'.

Re: [PATCH 2/2] perf record: Add --dry-run option to check cmdline options

[Wangnan (F)]

On 2016/6/21 0:22, Alexei Starovoitov wrote:
> On Mon, Jun 20, 2016 at 11:38:18AM -0300, Arnaldo Carvalho de Melo wrote:
>> Em Mon, Jun 20, 2016 at 11:29:13AM +0800, Wangnan (F) escreveu:
>>> On 2016/6/17 0:48, Arnaldo Carvalho de Melo wrote:
>>>> Em Thu, Jun 16, 2016 at 08:02:41AM +0000, Wang Nan escreveu:
>>>>> With '--dry-run', 'perf record' doesn't do reall recording. Combine with
>>>>> llvm.dump-obj option, --dry-run can be used to help compile BPF objects for
>>>>> embedded platform.
>>>> So these are nice and have value, but can we have a subcommand to do all
>>>> this with an expressive name, Something like:
>>>>    perf bpfcc foo.c -o foo
>>>> or shorter:
>>>>    perf bcc foo.c -o foo
>>>> Just like one would use gcc or some other compiler to generate something
>>>> for later use?
>>> I'll try it today. I thought a subcommand require a bigger feature,
>>> and wrapping clang is not big enough.
>> Not really, we may have as many as we like, given that they provide
>> something useful, like I think is the case here.
>>
>> Having to edit ~/.perfconfig, create a new section, a variable in it
>> with a boolean value (at first, just reading the changeset comment, I
>> thought I had to provide a directory where to store the objects
>> "dumped"), to then use a tool to record a .c event, but not recording
>> (use dry-run, which is useful to test the command line, etc), to then
>> get, on the current directory, the end result looked to me a convoluted
>> way to ask perf to compile the given .c file into a .o for later use.
>>
>> Doing:
>>
>> 	perf bcc -c foo.c
>>
>> Looks so much simpler and similar to an existing compile source code
>> into object file workflow (gcc's, any C compiler) that I think it would
>> fit in the workflow being discussed really nicely.
> I'm hopeful that eventually we'll be able merge iovisor/bcc project
> with perf, so would be good to reserve 'perf bcc' command for that
> future use. Also picking a different name for compiling would be less
> confusing to users who already familiar with bcc. Instead we can use:
> perf bpfcc foo.c -o foo.o
> perf cc foo.c
> perf compile foo.c
>

I think finally we should make perf independent with LLVM runtime.
I suggest 'perf bpf' subcommand to deal with all BPF related things,
include compiling, configuration and potential cache.

Thank you.

perf cc/perf bpf was: Re: [PATCH 2/2] perf record: Add --dry-run option to check cmdline options

[Arnaldo Carvalho de Melo]

Em Tue, Jun 21, 2016 at 02:12:38PM +0800, Wangnan (F) escreveu:
> On 2016/6/21 0:22, Alexei Starovoitov wrote:
> > On Mon, Jun 20, 2016 at 11:38:18AM -0300, Arnaldo Carvalho de Melo wrote:
> > > Em Mon, Jun 20, 2016 at 11:29:13AM +0800, Wangnan (F) escreveu:
> > > > On 2016/6/17 0:48, Arnaldo Carvalho de Melo wrote:
> > > 	perf bcc -c foo.c

> > > Looks so much simpler and similar to an existing compile source code
> > > into object file workflow (gcc's, any C compiler) that I think it would
> > > fit in the workflow being discussed really nicely.

> > I'm hopeful that eventually we'll be able merge iovisor/bcc project
> > with perf, so would be good to reserve 'perf bcc' command for that
> > future use. Also picking a different name for compiling would be less
> > confusing to users who already familiar with bcc. Instead we can use:

> > perf bpfcc foo.c -o foo.o
> > perf cc foo.c
> > perf compile foo.c
 
> I think finally we should make perf independent with LLVM runtime.
> I suggest 'perf bpf' subcommand to deal with all BPF related things,
> include compiling, configuration and potential cache.

I think 'bpf' is tied to a detail, the fact that right now it generates
ELF binaries that are loaded via sys_bpf(), but it is not necessarily a
filter (the F in BPF) nor deals exclusively with packets, etc.

What we're doing here? Getting a .c file and turning it into a .o file
suitable to be run. And doing it thru a compiler, why not call it that,
i.e. 'cc'?

If we end up supporting the compilation of .c code to some binary format
to attach to some other event, perhaps not even in the kernel or via
sys_bpf, we could just add that other format to 'perf cc', again reusing
the existing model of 'cc' generating ELF, a.out, etc.

- Arnaldo

[tip:perf/core] perf llvm: Allow dump llvm output object file using llvm.dump-obj

[tip-bot for Wang Nan]

Commit-ID:  f078464925f5b5c977c1196c67cae49cd82f40ff
Gitweb:     http://git.kernel.org/tip/f078464925f5b5c977c1196c67cae49cd82f40ff
Author:     Wang Nan <wangnan0@huawei.com>
AuthorDate: Thu, 16 Jun 2016 08:02:40 +0000
Committer:  Arnaldo Carvalho de Melo <acme@redhat.com>
CommitDate: Tue, 21 Jun 2016 13:18:34 -0300

perf llvm: Allow dump llvm output object file using llvm.dump-obj

Add a 'llvm.dump-obj' config option to enable perf dump BPF object files
compiled by LLVM.

This option is useful when using BPF objects in embedded platforms.
LLVM compiler won't be deployed in these platforms, and currently we
don't support dynamic compiling library.

Before this patch users have to explicitly issue llvm commands to
compile BPF scripts, and can't use helpers (like include path detection
and default macros) in perf. With this option, user is allowed to use
perf to compile their BPF objects then copy them into their embedded
platforms.

Committer notice:

Testing it:

  # cat ~/.perfconfig
  [llvm]
	dump-obj = true
  #
  # ls -la filter.o
  ls: cannot access filter.o: No such file or directory
  # cat filter.c
  #include <uapi/linux/bpf.h>
  #define SEC(NAME) __attribute__((section(NAME), used))

  SEC("func=hrtimer_nanosleep rqtp->tv_nsec")
  int func(void *ctx, int err, long nsec)
  {
	return nsec > 1000;
  }
  char _license[] SEC("license") = "GPL";
  int _version SEC("version") = LINUX_VERSION_CODE;
  # trace -e nanosleep --event filter.c usleep 6
  LLVM: dumping filter.o
     0.007 ( 0.007 ms): usleep/13976 nanosleep(rqtp: 0x7ffc5847f640                                        ) ...
     0.007 (         ): perf_bpf_probe:func:(ffffffff811137d0) tv_nsec=6000)
     0.070 ( 0.070 ms): usleep/13976  ... [continued]: nanosleep()) = 0
  # ls -la filter.o
  -rw-r--r--. 1 root root 776 Jun 20 17:01 filter.o
  # readelf -SW filter.o
  There are 7 section headers, starting at offset 0x148:

  Section Headers:
   [Nr] Name        Type       Address          Off    Size   ES Flg Lk Inf Al
   [ 0]             NULL       0000000000000000 000000 000000 00      0   0  0
   [ 1] .strtab     STRTAB     0000000000000000 0000e8 00005a 00      0   0  1
   [ 2] .text       PROGBITS   0000000000000000 000040 000000 00  AX  0   0  4
   [ 3] func=hrtimer_nanosleep rqtp->tv_nsec PROGBITS 0000000000000000 000040 000028 00  AX  0   0  8
   [ 4] license     PROGBITS   0000000000000000 000068 000004 00  WA  0   0  1
   [ 5] version     PROGBITS   0000000000000000 00006c 000004 00  WA  0   0  4
   [ 6] .symtab     SYMTAB     0000000000000000 000070 000078 18      1   2  8
  Key to Flags:
   W (write), A (alloc), X (execute), M (merge), S (strings)
   I (info), L (link order), G (group), T (TLS), E (exclude), x (unknown)
   O (extra OS processing required) o (OS specific), p (processor specific)
   #

Signed-off-by: Wang Nan <wangnan0@huawei.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Jiri Olsa <jolsa@kernel.org>
Cc: pi3orama@163.com
Link: http://lkml.kernel.org/r/1466064161-48553-2-git-send-email-wangnan0@huawei.com
[ s/dumpping/dumping/g ]
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
---
 tools/perf/util/llvm-utils.c | 42 ++++++++++++++++++++++++++++++++++++++++++
 tools/perf/util/llvm-utils.h |  5 +++++
 2 files changed, 47 insertions(+)

diff --git a/tools/perf/util/llvm-utils.c b/tools/perf/util/llvm-utils.c
index 33071d6..878a566 100644
--- a/tools/perf/util/llvm-utils.c
+++ b/tools/perf/util/llvm-utils.c
@@ -42,6 +42,8 @@ int perf_llvm_config(const char *var, const char *value)
 		llvm_param.kbuild_dir = strdup(value);
 	else if (!strcmp(var, "kbuild-opts"))
 		llvm_param.kbuild_opts = strdup(value);
+	else if (!strcmp(var, "dump-obj"))
+		llvm_param.dump_obj = !!perf_config_bool(var, value);
 	else
 		return -1;
 	llvm_param.user_set_param = true;
@@ -326,6 +328,42 @@ get_kbuild_opts(char **kbuild_dir, char **kbuild_include_opts)
 	pr_debug("include option is set to %s\n", *kbuild_include_opts);
 }
 
+static void
+dump_obj(const char *path, void *obj_buf, size_t size)
+{
+	char *obj_path = strdup(path);
+	FILE *fp;
+	char *p;
+
+	if (!obj_path) {
+		pr_warning("WARNING: No enough memory, skip object dumping\n");
+		return;
+	}
+
+	p = strrchr(obj_path, '.');
+	if (!p || (strcmp(p, ".c") != 0)) {
+		pr_warning("WARNING: invalid llvm source path: '%s', skip object dumping\n",
+			   obj_path);
+		goto out;
+	}
+
+	p[1] = 'o';
+	fp = fopen(obj_path, "wb");
+	if (!fp) {
+		pr_warning("WARNING: failed to open '%s': %s, skip object dumping\n",
+			   obj_path, strerror(errno));
+		goto out;
+	}
+
+	pr_info("LLVM: dumping %s\n", obj_path);
+	if (fwrite(obj_buf, size, 1, fp) != 1)
+		pr_warning("WARNING: failed to write to file '%s': %s, skip object dumping\n",
+			   obj_path, strerror(errno));
+	fclose(fp);
+out:
+	free(obj_path);
+}
+
 int llvm__compile_bpf(const char *path, void **p_obj_buf,
 		      size_t *p_obj_buf_sz)
 {
@@ -411,6 +449,10 @@ int llvm__compile_bpf(const char *path, void **p_obj_buf,
 
 	free(kbuild_dir);
 	free(kbuild_include_opts);
+
+	if (llvm_param.dump_obj)
+		dump_obj(path, obj_buf, obj_buf_sz);
+
 	if (!p_obj_buf)
 		free(obj_buf);
 	else
diff --git a/tools/perf/util/llvm-utils.h b/tools/perf/util/llvm-utils.h
index 23b9a74..9f501ce 100644
--- a/tools/perf/util/llvm-utils.h
+++ b/tools/perf/util/llvm-utils.h
@@ -30,6 +30,11 @@ struct llvm_param {
 	 */
 	const char *kbuild_opts;
 	/*
+	 * Default is false. If set to true, write compiling result
+	 * to object file.
+	 */
+	bool dump_obj;
+	/*
 	 * Default is false. If one of the above fields is set by user
 	 * explicitly then user_set_llvm is set to true. This is used
 	 * for perf test. If user doesn't set anything in .perfconfig

[tip:perf/core] perf record: Add --dry-run option to check cmdline options

[tip-bot for Wang Nan]

Commit-ID:  0aab21363ffa66d6e7340bc50cc5bfae865fd1a6
Gitweb:     http://git.kernel.org/tip/0aab21363ffa66d6e7340bc50cc5bfae865fd1a6
Author:     Wang Nan <wangnan0@huawei.com>
AuthorDate: Thu, 16 Jun 2016 08:02:41 +0000
Committer:  Arnaldo Carvalho de Melo <acme@redhat.com>
CommitDate: Tue, 21 Jun 2016 13:18:35 -0300

perf record: Add --dry-run option to check cmdline options

With '--dry-run', 'perf record' doesn't do reall recording. Combine with
llvm.dump-obj option, --dry-run can be used to help compile BPF objects
for embedded platform.

Signed-off-by: Wang Nan <wangnan0@huawei.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Jiri Olsa <jolsa@kernel.org>
Cc: pi3orama@163.com
Link: http://lkml.kernel.org/r/1466064161-48553-3-git-send-email-wangnan0@huawei.com
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
---
 tools/perf/Documentation/perf-record.txt | 7 +++++++
 tools/perf/builtin-record.c              | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt
index 8dbee83..5b46b1d 100644
--- a/tools/perf/Documentation/perf-record.txt
+++ b/tools/perf/Documentation/perf-record.txt
@@ -360,6 +360,13 @@ particular perf.data snapshot should be kept or not.
 
 Implies --timestamp-filename, --no-buildid and --no-buildid-cache.
 
+--dry-run::
+Parse options then exit. --dry-run can be used to detect errors in cmdline
+options.
+
+'perf record --dry-run -e' can act as a BPF script compiler if llvm.dump-obj
+in config file is set to true.
+
 SEE ALSO
 --------
 linkperf:perf-stat[1], linkperf:perf-list[1]
diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
index d4cf1b0..b1304eb 100644
--- a/tools/perf/builtin-record.c
+++ b/tools/perf/builtin-record.c
@@ -1274,6 +1274,8 @@ static struct record record = {
 const char record_callchain_help[] = CALLCHAIN_RECORD_HELP
 	"\n\t\t\t\tDefault: fp";
 
+static bool dry_run;
+
 /*
  * XXX Will stay a global variable till we fix builtin-script.c to stop messing
  * with it and switch to use the library functions in perf_evlist that came
@@ -1393,6 +1395,8 @@ struct option __record_options[] = {
 		    "append timestamp to output filename"),
 	OPT_BOOLEAN(0, "switch-output", &record.switch_output,
 		    "Switch output when receive SIGUSR2"),
+	OPT_BOOLEAN(0, "dry-run", &dry_run,
+		    "Parse options then exit"),
 	OPT_END()
 };
 
@@ -1462,6 +1466,9 @@ int cmd_record(int argc, const char **argv, const char *prefix __maybe_unused)
 	if (err)
 		return err;
 
+	if (dry_run)
+		return 0;
+
 	err = bpf__setup_stdout(rec->evlist);
 	if (err) {
 		bpf__strerror_setup_stdout(rec->evlist, err, errbuf, sizeof(errbuf));