[PATCH 0/3] perf probe: Boot time tracing support

[PATCH 0/3] perf probe: Boot time tracing support

[Masami Hiramatsu]

Hi,

Here is a series of patches to add boot-time tracing support to perf
probe command.
Recently, I tried to debug kernel boot process with boot-time tracing
and I found perf probe lacks some features for it.

[1/3] fixes perf probe to show probe definitions on __init functions.
Since __init functions are removed after boot (and those are not in
.text but .inittext section), perf's map object can not find them.
[2/3] is a code cleanup preparing for the next patch.
[3/3] adds --bootconfig option for showing the probe definitions in
the bootconfig format. This option can be used with -D (--definition)
option.

One todo is improving -F option. Since the -F option also using the
map object, it can not list up the __init function. Anyway, that may
not be important. Usually, the person who debugs kernel already know
the target function name, and can use "perf probe -L func" for
checking the function source code. (Note that perf probe -L doesn't
depend on map object)

Thank you,

---

Masami Hiramatsu (3):
      perf/probe: Support probes on init functions for offline kernel
      perf/probe: Cleanup synthesize_probe_trace_command
      perf/probe: Add --bootconfig to output definition in bootconfig format


 tools/perf/builtin-probe.c    |   12 +++
 tools/perf/util/probe-event.c |  167 +++++++++++++++++++++++++++++++----------
 tools/perf/util/probe-event.h |    2 
 3 files changed, 141 insertions(+), 40 deletions(-)

--
Masami Hiramatsu (Linaro) <mhiramat@kernel.org>

[PATCH 1/3] perf/probe: Support probes on init functions for offline kernel

[Masami Hiramatsu]

perf probe internally checks the probe target is in the text area
in post-process (after analyzing debuginfo). But it fails if the
probe target is in the "inittext".
This is a good limitation for the online kernel because such
functions have gone after booting. However, for using it for
boot-time tracing, user may want to put a probe on init functions.

This skips the post checking process if the target is offline kenrel
so that user can get the probe definition on the init functions.

Without this patch:
  $ perf probe -k ./build-x86_64/vmlinux -D do_mount_root:10
  Probe point 'do_mount_root:10' not found.
    Error: Failed to add events.

With this patch:
  $ perf probe -k ./build-x86_64/vmlinux -D do_mount_root:10
  p:probe/do_mount_root_L10 mount_block_root+300


Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org>
---
 tools/perf/util/probe-event.c |    9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c
index 8fe179d671c3..a6e121afb651 100644
--- a/tools/perf/util/probe-event.c
+++ b/tools/perf/util/probe-event.c
@@ -682,8 +682,13 @@ static int post_process_probe_trace_point(struct probe_trace_point *tp,
 	u64 addr = tp->address - offs;
 
 	sym = map__find_symbol(map, addr);
-	if (!sym)
-		return -ENOENT;
+	if (!sym) {
+		/*
+		 * If the address is in the inittext section, map can not
+		 * find it. Ignore it if we are probing offline kernel.
+		 */
+		return (symbol_conf.ignore_vmlinux_buildid) ? 0 : -ENOENT;
+	}
 
 	if (strcmp(sym->name, tp->symbol)) {
 		/* If we have no realname, use symbol for it */

[PATCH 2/3] perf/probe: Cleanup synthesize_probe_trace_command

[Masami Hiramatsu]

Cleanup the synthesize_probe_trace_command() to simplify
code path.

Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org>
---
 tools/perf/util/probe-event.c |   86 +++++++++++++++++++++++------------------
 1 file changed, 49 insertions(+), 37 deletions(-)

diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c
index a6e121afb651..505c0702dbe2 100644
--- a/tools/perf/util/probe-event.c
+++ b/tools/perf/util/probe-event.c
@@ -2127,19 +2127,55 @@ static int synthesize_probe_trace_arg(struct probe_trace_arg *arg,
 }
 
 static int
-synthesize_uprobe_trace_def(struct probe_trace_event *tev, struct strbuf *buf)
+synthesize_probe_trace_args(struct probe_trace_event *tev, struct strbuf *buf)
+{
+	int i, ret = 0;
+
+	for (i = 0; i < tev->nargs && ret >= 0; i++)
+		ret = synthesize_probe_trace_arg(&tev->args[i], buf);
+
+	return ret;
+}
+
+static int
+synthesize_uprobe_trace_def(struct probe_trace_point *tp, struct strbuf *buf)
 {
-	struct probe_trace_point *tp = &tev->point;
 	int err;
 
+	/* Uprobes must have tp->module */
+	if (!tp->module)
+		return -EINVAL;
+	/*
+	 * If tp->address == 0, then this point must be a
+	 * absolute address uprobe.
+	 * try_to_find_absolute_address() should have made
+	 * tp->symbol to "0x0".
+	 */
+	if (!tp->address && (!tp->symbol || strcmp(tp->symbol, "0x0")))
+		return -EINVAL;
+
+	/* Use the tp->address for uprobes */
 	err = strbuf_addf(buf, "%s:0x%lx", tp->module, tp->address);
 
 	if (err >= 0 && tp->ref_ctr_offset) {
 		if (!uprobe_ref_ctr_is_supported())
-			return -1;
+			return -EINVAL;
 		err = strbuf_addf(buf, "(0x%lx)", tp->ref_ctr_offset);
 	}
-	return err >= 0 ? 0 : -1;
+	return err >= 0 ? 0 : err;
+}
+
+static int
+synthesize_kprobe_trace_def(struct probe_trace_point *tp, struct strbuf *buf)
+{
+	if (!strncmp(tp->symbol, "0x", 2)) {
+		/* Absolute address. See try_to_find_absolute_address() */
+		return strbuf_addf(buf, "%s%s0x%lx", tp->module ?: "",
+				  tp->module ? ":" : "", tp->address);
+	} else {
+		return strbuf_addf(buf, "%s%s%s+%lu", tp->module ?: "",
+				tp->module ? ":" : "", tp->symbol, tp->offset);
+	}
 }
 
 char *synthesize_probe_trace_command(struct probe_trace_event *tev)
@@ -2147,11 +2183,7 @@ char *synthesize_probe_trace_command(struct probe_trace_event *tev)
 	struct probe_trace_point *tp = &tev->point;
 	struct strbuf buf;
 	char *ret = NULL;
-	int i, err;
-
-	/* Uprobes must have tp->module */
-	if (tev->uprobes && !tp->module)
-		return NULL;
+	int err;
 
 	if (strbuf_init(&buf, 32) < 0)
 		return NULL;
@@ -2159,37 +2191,17 @@ char *synthesize_probe_trace_command(struct probe_trace_event *tev)
 	if (strbuf_addf(&buf, "%c:%s/%s ", tp->retprobe ? 'r' : 'p',
 			tev->group, tev->event) < 0)
 		goto error;
-	/*
-	 * If tp->address == 0, then this point must be a
-	 * absolute address uprobe.
-	 * try_to_find_absolute_address() should have made
-	 * tp->symbol to "0x0".
-	 */
-	if (tev->uprobes && !tp->address) {
-		if (!tp->symbol || strcmp(tp->symbol, "0x0"))
-			goto error;
-	}
 
-	/* Use the tp->address for uprobes */
-	if (tev->uprobes) {
-		err = synthesize_uprobe_trace_def(tev, &buf);
-	} else if (!strncmp(tp->symbol, "0x", 2)) {
-		/* Absolute address. See try_to_find_absolute_address() */
-		err = strbuf_addf(&buf, "%s%s0x%lx", tp->module ?: "",
-				  tp->module ? ":" : "", tp->address);
-	} else {
-		err = strbuf_addf(&buf, "%s%s%s+%lu", tp->module ?: "",
-				tp->module ? ":" : "", tp->symbol, tp->offset);
-	}
-
-	if (err)
-		goto error;
+	if (tev->uprobes)
+		err = synthesize_uprobe_trace_def(tp, &buf);
+	else
+		err = synthesize_kprobe_trace_def(tp, &buf);
 
-	for (i = 0; i < tev->nargs; i++)
-		if (synthesize_probe_trace_arg(&tev->args[i], &buf) < 0)
-			goto error;
+	if (err >= 0)
+		err = synthesize_probe_trace_args(tev, &buf);
 
-	ret = strbuf_detach(&buf, NULL);
+	if (err >= 0)
+		ret = strbuf_detach(&buf, NULL);
 error:
 	strbuf_release(&buf);
 	return ret;

[PATCH 3/3] perf/probe: Add --bootconfig to output definition in bootconfig format

[Masami Hiramatsu]

Now the boot-time tracing supports the kprobes events and that
must be written in bootconfig file as following format.

ftrace.event.kprobes.<EVENT_NAME>.probes = <PROBE-DEF>

The perf probe already supports --definition (-D) action to
show the probe definitions, but the format is for the tracefs.

[p|r][:EVENT_NAME] <PROBE-DEF>

This adds --bootconfig option for -D action so that it output
the probe definitions in bootconfig fromat. E.g.

 $ perf probe --bootconfig -D "path_lookupat:7 err:s32 s:string"
 ftrace.event.kprobes.path_lookupat_L7.probe = 'path_lookupat.isra.0+309 err_s32=%ax:s32 s_string=+0(%r13):string'

Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org>
---
 tools/perf/builtin-probe.c    |   12 ++++++-
 tools/perf/util/probe-event.c |   72 +++++++++++++++++++++++++++++++++++++++++
 tools/perf/util/probe-event.h |    2 +
 3 files changed, 85 insertions(+), 1 deletion(-)

diff --git a/tools/perf/builtin-probe.c b/tools/perf/builtin-probe.c
index 6b1507566770..2bfd41df621c 100644
--- a/tools/perf/builtin-probe.c
+++ b/tools/perf/builtin-probe.c
@@ -347,7 +347,10 @@ static int perf_add_probe_events(struct perf_probe_event *pevs, int npevs)
 		goto out_cleanup;
 
 	if (params.command == 'D') {	/* it shows definition */
-		ret = show_probe_trace_events(pevs, npevs);
+		if (probe_conf.bootconfig)
+			ret = show_bootconfig_events(pevs, npevs);
+		else
+			ret = show_probe_trace_events(pevs, npevs);
 		goto out_cleanup;
 	}
 
@@ -581,6 +584,8 @@ __cmd_probe(int argc, const char **argv)
 		   "Look for files with symbols relative to this directory"),
 	OPT_CALLBACK(0, "target-ns", NULL, "pid",
 		     "target pid for namespace contexts", opt_set_target_ns),
+	OPT_BOOLEAN(0, "bootconfig", &probe_conf.bootconfig,
+		    "Output probe definition with bootconfig format"),
 	OPT_END()
 	};
 	int ret;
@@ -692,6 +697,11 @@ __cmd_probe(int argc, const char **argv)
 		}
 		break;
 	case 'D':
+		if (probe_conf.bootconfig && params.uprobes) {
+			pr_err("  Error: --bootconfig doesn't support uprobes.\n");
+			return -EINVAL;
+		}
+		__fallthrough;
 	case 'a':
 
 		/* Ensure the last given target is used */
diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c
index 505c0702dbe2..f1348fa3dd1b 100644
--- a/tools/perf/util/probe-event.c
+++ b/tools/perf/util/probe-event.c
@@ -3564,6 +3564,78 @@ int show_probe_trace_events(struct perf_probe_event *pevs, int npevs)
 	return ret;
 }
 
+static int show_bootconfig_event(struct probe_trace_event *tev)
+{
+	struct probe_trace_point *tp = &tev->point;
+	struct strbuf buf;
+	char *ret = NULL;
+	int err;
+
+	if (strbuf_init(&buf, 32) < 0)
+		return -ENOMEM;
+
+	err = synthesize_kprobe_trace_def(tp, &buf);
+	if (err >= 0)
+		err = synthesize_probe_trace_args(tev, &buf);
+	if (err >= 0)
+		ret = strbuf_detach(&buf, NULL);
+	strbuf_release(&buf);
+
+	if (ret) {
+		printf("'%s'", ret);
+		free(ret);
+	}
+
+	return err;
+}
+
+int show_bootconfig_events(struct perf_probe_event *pevs, int npevs)
+{
+	struct strlist *namelist = strlist__new(NULL, NULL);
+	struct probe_trace_event *tev;
+	struct perf_probe_event *pev;
+	char *cur_name = NULL;
+	int i, j, ret = 0;
+
+	if (!namelist)
+		return -ENOMEM;
+
+	for (j = 0; j < npevs && !ret; j++) {
+		pev = &pevs[j];
+		if (pev->group && strcmp(pev->group, "probe"))
+			pr_warning("WARN: Group name %s is ignored\n", pev->group);
+		if (pev->uprobes) {
+			pr_warning("ERROR: Bootconfig doesn't support uprobes\n");
+			ret = -EINVAL;
+			break;
+		}
+		for (i = 0; i < pev->ntevs && !ret; i++) {
+			tev = &pev->tevs[i];
+			/* Skip if the symbol is out of .text or blacklisted */
+			if (!tev->point.symbol && !pev->uprobes)
+				continue;
+
+			/* Set new name for tev (and update namelist) */
+			ret = probe_trace_event__set_name(tev, pev,
+							  namelist, true);
+			if (ret)
+				break;
+
+			if (!cur_name || strcmp(cur_name, tev->event)) {
+				printf("%sftrace.event.kprobes.%s.probe = ",
+					cur_name ? "\n" : "", tev->event);
+				cur_name = tev->event;
+			} else
+				printf(", ");
+			ret = show_bootconfig_event(tev);
+		}
+	}
+	printf("\n");
+	strlist__delete(namelist);
+
+	return ret;
+}
+
 int apply_perf_probe_events(struct perf_probe_event *pevs, int npevs)
 {
 	int i, ret = 0;
diff --git a/tools/perf/util/probe-event.h b/tools/perf/util/probe-event.h
index 4f0eb3a20c36..65769d7949a3 100644
--- a/tools/perf/util/probe-event.h
+++ b/tools/perf/util/probe-event.h
@@ -15,6 +15,7 @@ struct probe_conf {
 	bool	force_add;
 	bool	no_inlines;
 	bool	cache;
+	bool	bootconfig;
 	int	max_probes;
 	unsigned long	magic_num;
 };
@@ -163,6 +164,7 @@ int add_perf_probe_events(struct perf_probe_event *pevs, int npevs);
 int convert_perf_probe_events(struct perf_probe_event *pevs, int npevs);
 int apply_perf_probe_events(struct perf_probe_event *pevs, int npevs);
 int show_probe_trace_events(struct perf_probe_event *pevs, int npevs);
+int show_bootconfig_events(struct perf_probe_event *pevs, int npevs);
 void cleanup_perf_probe_events(struct perf_probe_event *pevs, int npevs);
 
 struct strfilter;

Re: [PATCH 3/3] perf/probe: Add --bootconfig to output definition in bootconfig format

[Arnaldo Carvalho de Melo]

Em Sat, Jun 05, 2021 at 01:28:43AM +0900, Masami Hiramatsu escreveu:
> Now the boot-time tracing supports the kprobes events and that
> must be written in bootconfig file as following format.
> 
> ftrace.event.kprobes.<EVENT_NAME>.probes = <PROBE-DEF>
> 
> The perf probe already supports --definition (-D) action to
> show the probe definitions, but the format is for the tracefs.

You forgot to add the tools/perf/Documentation/perf-probe.txt entry for
this new command line option, please sent it as a followup patch.

Applied.

- Arnaldo
 
> [p|r][:EVENT_NAME] <PROBE-DEF>
> 
> This adds --bootconfig option for -D action so that it output
> the probe definitions in bootconfig fromat. E.g.
> 
>  $ perf probe --bootconfig -D "path_lookupat:7 err:s32 s:string"
>  ftrace.event.kprobes.path_lookupat_L7.probe = 'path_lookupat.isra.0+309 err_s32=%ax:s32 s_string=+0(%r13):string'
> 
> Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org>
> ---
>  tools/perf/builtin-probe.c    |   12 ++++++-
>  tools/perf/util/probe-event.c |   72 +++++++++++++++++++++++++++++++++++++++++
>  tools/perf/util/probe-event.h |    2 +
>  3 files changed, 85 insertions(+), 1 deletion(-)
> 
> diff --git a/tools/perf/builtin-probe.c b/tools/perf/builtin-probe.c
> index 6b1507566770..2bfd41df621c 100644
> --- a/tools/perf/builtin-probe.c
> +++ b/tools/perf/builtin-probe.c
> @@ -347,7 +347,10 @@ static int perf_add_probe_events(struct perf_probe_event *pevs, int npevs)
>  		goto out_cleanup;
>  
>  	if (params.command == 'D') {	/* it shows definition */
> -		ret = show_probe_trace_events(pevs, npevs);
> +		if (probe_conf.bootconfig)
> +			ret = show_bootconfig_events(pevs, npevs);
> +		else
> +			ret = show_probe_trace_events(pevs, npevs);
>  		goto out_cleanup;
>  	}
>  
> @@ -581,6 +584,8 @@ __cmd_probe(int argc, const char **argv)
>  		   "Look for files with symbols relative to this directory"),
>  	OPT_CALLBACK(0, "target-ns", NULL, "pid",
>  		     "target pid for namespace contexts", opt_set_target_ns),
> +	OPT_BOOLEAN(0, "bootconfig", &probe_conf.bootconfig,
> +		    "Output probe definition with bootconfig format"),
>  	OPT_END()
>  	};
>  	int ret;
> @@ -692,6 +697,11 @@ __cmd_probe(int argc, const char **argv)
>  		}
>  		break;
>  	case 'D':
> +		if (probe_conf.bootconfig && params.uprobes) {
> +			pr_err("  Error: --bootconfig doesn't support uprobes.\n");
> +			return -EINVAL;
> +		}
> +		__fallthrough;
>  	case 'a':
>  
>  		/* Ensure the last given target is used */
> diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c
> index 505c0702dbe2..f1348fa3dd1b 100644
> --- a/tools/perf/util/probe-event.c
> +++ b/tools/perf/util/probe-event.c
> @@ -3564,6 +3564,78 @@ int show_probe_trace_events(struct perf_probe_event *pevs, int npevs)
>  	return ret;
>  }
>  
> +static int show_bootconfig_event(struct probe_trace_event *tev)
> +{
> +	struct probe_trace_point *tp = &tev->point;
> +	struct strbuf buf;
> +	char *ret = NULL;
> +	int err;
> +
> +	if (strbuf_init(&buf, 32) < 0)
> +		return -ENOMEM;
> +
> +	err = synthesize_kprobe_trace_def(tp, &buf);
> +	if (err >= 0)
> +		err = synthesize_probe_trace_args(tev, &buf);
> +	if (err >= 0)
> +		ret = strbuf_detach(&buf, NULL);
> +	strbuf_release(&buf);
> +
> +	if (ret) {
> +		printf("'%s'", ret);
> +		free(ret);
> +	}
> +
> +	return err;
> +}
> +
> +int show_bootconfig_events(struct perf_probe_event *pevs, int npevs)
> +{
> +	struct strlist *namelist = strlist__new(NULL, NULL);
> +	struct probe_trace_event *tev;
> +	struct perf_probe_event *pev;
> +	char *cur_name = NULL;
> +	int i, j, ret = 0;
> +
> +	if (!namelist)
> +		return -ENOMEM;
> +
> +	for (j = 0; j < npevs && !ret; j++) {
> +		pev = &pevs[j];
> +		if (pev->group && strcmp(pev->group, "probe"))
> +			pr_warning("WARN: Group name %s is ignored\n", pev->group);
> +		if (pev->uprobes) {
> +			pr_warning("ERROR: Bootconfig doesn't support uprobes\n");
> +			ret = -EINVAL;
> +			break;
> +		}
> +		for (i = 0; i < pev->ntevs && !ret; i++) {
> +			tev = &pev->tevs[i];
> +			/* Skip if the symbol is out of .text or blacklisted */
> +			if (!tev->point.symbol && !pev->uprobes)
> +				continue;
> +
> +			/* Set new name for tev (and update namelist) */
> +			ret = probe_trace_event__set_name(tev, pev,
> +							  namelist, true);
> +			if (ret)
> +				break;
> +
> +			if (!cur_name || strcmp(cur_name, tev->event)) {
> +				printf("%sftrace.event.kprobes.%s.probe = ",
> +					cur_name ? "\n" : "", tev->event);
> +				cur_name = tev->event;
> +			} else
> +				printf(", ");
> +			ret = show_bootconfig_event(tev);
> +		}
> +	}
> +	printf("\n");
> +	strlist__delete(namelist);
> +
> +	return ret;
> +}
> +
>  int apply_perf_probe_events(struct perf_probe_event *pevs, int npevs)
>  {
>  	int i, ret = 0;
> diff --git a/tools/perf/util/probe-event.h b/tools/perf/util/probe-event.h
> index 4f0eb3a20c36..65769d7949a3 100644
> --- a/tools/perf/util/probe-event.h
> +++ b/tools/perf/util/probe-event.h
> @@ -15,6 +15,7 @@ struct probe_conf {
>  	bool	force_add;
>  	bool	no_inlines;
>  	bool	cache;
> +	bool	bootconfig;
>  	int	max_probes;
>  	unsigned long	magic_num;
>  };
> @@ -163,6 +164,7 @@ int add_perf_probe_events(struct perf_probe_event *pevs, int npevs);
>  int convert_perf_probe_events(struct perf_probe_event *pevs, int npevs);
>  int apply_perf_probe_events(struct perf_probe_event *pevs, int npevs);
>  int show_probe_trace_events(struct perf_probe_event *pevs, int npevs);
> +int show_bootconfig_events(struct perf_probe_event *pevs, int npevs);
>  void cleanup_perf_probe_events(struct perf_probe_event *pevs, int npevs);
>  
>  struct strfilter;
> 

-- 

- Arnaldo

Re: [PATCH 3/3] perf/probe: Add --bootconfig to output definition in bootconfig format

[Masami Hiramatsu]

On Fri, 18 Jun 2021 10:14:02 -0300
Arnaldo Carvalho de Melo <acme@kernel.org> wrote:

> Em Sat, Jun 05, 2021 at 01:28:43AM +0900, Masami Hiramatsu escreveu:
> > Now the boot-time tracing supports the kprobes events and that
> > must be written in bootconfig file as following format.
> > 
> > ftrace.event.kprobes.<EVENT_NAME>.probes = <PROBE-DEF>
> > 
> > The perf probe already supports --definition (-D) action to
> > show the probe definitions, but the format is for the tracefs.
> 
> You forgot to add the tools/perf/Documentation/perf-probe.txt entry for
> this new command line option, please sent it as a followup patch.

Oops, let me write it up! Thanks for pointing!

> 
> Applied.
> 
> - Arnaldo
>  
> > [p|r][:EVENT_NAME] <PROBE-DEF>
> > 
> > This adds --bootconfig option for -D action so that it output
> > the probe definitions in bootconfig fromat. E.g.
> > 
> >  $ perf probe --bootconfig -D "path_lookupat:7 err:s32 s:string"
> >  ftrace.event.kprobes.path_lookupat_L7.probe = 'path_lookupat.isra.0+309 err_s32=%ax:s32 s_string=+0(%r13):string'
> > 
> > Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org>
> > ---
> >  tools/perf/builtin-probe.c    |   12 ++++++-
> >  tools/perf/util/probe-event.c |   72 +++++++++++++++++++++++++++++++++++++++++
> >  tools/perf/util/probe-event.h |    2 +
> >  3 files changed, 85 insertions(+), 1 deletion(-)
> > 
> > diff --git a/tools/perf/builtin-probe.c b/tools/perf/builtin-probe.c
> > index 6b1507566770..2bfd41df621c 100644
> > --- a/tools/perf/builtin-probe.c
> > +++ b/tools/perf/builtin-probe.c
> > @@ -347,7 +347,10 @@ static int perf_add_probe_events(struct perf_probe_event *pevs, int npevs)
> >  		goto out_cleanup;
> >  
> >  	if (params.command == 'D') {	/* it shows definition */
> > -		ret = show_probe_trace_events(pevs, npevs);
> > +		if (probe_conf.bootconfig)
> > +			ret = show_bootconfig_events(pevs, npevs);
> > +		else
> > +			ret = show_probe_trace_events(pevs, npevs);
> >  		goto out_cleanup;
> >  	}
> >  
> > @@ -581,6 +584,8 @@ __cmd_probe(int argc, const char **argv)
> >  		   "Look for files with symbols relative to this directory"),
> >  	OPT_CALLBACK(0, "target-ns", NULL, "pid",
> >  		     "target pid for namespace contexts", opt_set_target_ns),
> > +	OPT_BOOLEAN(0, "bootconfig", &probe_conf.bootconfig,
> > +		    "Output probe definition with bootconfig format"),
> >  	OPT_END()
> >  	};
> >  	int ret;
> > @@ -692,6 +697,11 @@ __cmd_probe(int argc, const char **argv)
> >  		}
> >  		break;
> >  	case 'D':
> > +		if (probe_conf.bootconfig && params.uprobes) {
> > +			pr_err("  Error: --bootconfig doesn't support uprobes.\n");
> > +			return -EINVAL;
> > +		}
> > +		__fallthrough;
> >  	case 'a':
> >  
> >  		/* Ensure the last given target is used */
> > diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c
> > index 505c0702dbe2..f1348fa3dd1b 100644
> > --- a/tools/perf/util/probe-event.c
> > +++ b/tools/perf/util/probe-event.c
> > @@ -3564,6 +3564,78 @@ int show_probe_trace_events(struct perf_probe_event *pevs, int npevs)
> >  	return ret;
> >  }
> >  
> > +static int show_bootconfig_event(struct probe_trace_event *tev)
> > +{
> > +	struct probe_trace_point *tp = &tev->point;
> > +	struct strbuf buf;
> > +	char *ret = NULL;
> > +	int err;
> > +
> > +	if (strbuf_init(&buf, 32) < 0)
> > +		return -ENOMEM;
> > +
> > +	err = synthesize_kprobe_trace_def(tp, &buf);
> > +	if (err >= 0)
> > +		err = synthesize_probe_trace_args(tev, &buf);
> > +	if (err >= 0)
> > +		ret = strbuf_detach(&buf, NULL);
> > +	strbuf_release(&buf);
> > +
> > +	if (ret) {
> > +		printf("'%s'", ret);
> > +		free(ret);
> > +	}
> > +
> > +	return err;
> > +}
> > +
> > +int show_bootconfig_events(struct perf_probe_event *pevs, int npevs)
> > +{
> > +	struct strlist *namelist = strlist__new(NULL, NULL);
> > +	struct probe_trace_event *tev;
> > +	struct perf_probe_event *pev;
> > +	char *cur_name = NULL;
> > +	int i, j, ret = 0;
> > +
> > +	if (!namelist)
> > +		return -ENOMEM;
> > +
> > +	for (j = 0; j < npevs && !ret; j++) {
> > +		pev = &pevs[j];
> > +		if (pev->group && strcmp(pev->group, "probe"))
> > +			pr_warning("WARN: Group name %s is ignored\n", pev->group);
> > +		if (pev->uprobes) {
> > +			pr_warning("ERROR: Bootconfig doesn't support uprobes\n");
> > +			ret = -EINVAL;
> > +			break;
> > +		}
> > +		for (i = 0; i < pev->ntevs && !ret; i++) {
> > +			tev = &pev->tevs[i];
> > +			/* Skip if the symbol is out of .text or blacklisted */
> > +			if (!tev->point.symbol && !pev->uprobes)
> > +				continue;
> > +
> > +			/* Set new name for tev (and update namelist) */
> > +			ret = probe_trace_event__set_name(tev, pev,
> > +							  namelist, true);
> > +			if (ret)
> > +				break;
> > +
> > +			if (!cur_name || strcmp(cur_name, tev->event)) {
> > +				printf("%sftrace.event.kprobes.%s.probe = ",
> > +					cur_name ? "\n" : "", tev->event);
> > +				cur_name = tev->event;
> > +			} else
> > +				printf(", ");
> > +			ret = show_bootconfig_event(tev);
> > +		}
> > +	}
> > +	printf("\n");
> > +	strlist__delete(namelist);
> > +
> > +	return ret;
> > +}
> > +
> >  int apply_perf_probe_events(struct perf_probe_event *pevs, int npevs)
> >  {
> >  	int i, ret = 0;
> > diff --git a/tools/perf/util/probe-event.h b/tools/perf/util/probe-event.h
> > index 4f0eb3a20c36..65769d7949a3 100644
> > --- a/tools/perf/util/probe-event.h
> > +++ b/tools/perf/util/probe-event.h
> > @@ -15,6 +15,7 @@ struct probe_conf {
> >  	bool	force_add;
> >  	bool	no_inlines;
> >  	bool	cache;
> > +	bool	bootconfig;
> >  	int	max_probes;
> >  	unsigned long	magic_num;
> >  };
> > @@ -163,6 +164,7 @@ int add_perf_probe_events(struct perf_probe_event *pevs, int npevs);
> >  int convert_perf_probe_events(struct perf_probe_event *pevs, int npevs);
> >  int apply_perf_probe_events(struct perf_probe_event *pevs, int npevs);
> >  int show_probe_trace_events(struct perf_probe_event *pevs, int npevs);
> > +int show_bootconfig_events(struct perf_probe_event *pevs, int npevs);
> >  void cleanup_perf_probe_events(struct perf_probe_event *pevs, int npevs);
> >  
> >  struct strfilter;
> > 
> 
> -- 
> 
> - Arnaldo


-- 
Masami Hiramatsu <mhiramat@kernel.org>

Re: [PATCH 3/3] perf/probe: Add --bootconfig to output definition in bootconfig format

[Masami Hiramatsu]

Hi,

On Sat,  5 Jun 2021 01:28:43 +0900
Masami Hiramatsu <mhiramat@kernel.org> wrote:

> +int show_bootconfig_events(struct perf_probe_event *pevs, int npevs)
> +{
> +	struct strlist *namelist = strlist__new(NULL, NULL);
> +	struct probe_trace_event *tev;
> +	struct perf_probe_event *pev;
> +	char *cur_name = NULL;
> +	int i, j, ret = 0;
> +
> +	if (!namelist)
> +		return -ENOMEM;
> +
> +	for (j = 0; j < npevs && !ret; j++) {
> +		pev = &pevs[j];
> +		if (pev->group && strcmp(pev->group, "probe"))
> +			pr_warning("WARN: Group name %s is ignored\n", pev->group);
> +		if (pev->uprobes) {
> +			pr_warning("ERROR: Bootconfig doesn't support uprobes\n");
> +			ret = -EINVAL;
> +			break;
> +		}
> +		for (i = 0; i < pev->ntevs && !ret; i++) {
> +			tev = &pev->tevs[i];
> +			/* Skip if the symbol is out of .text or blacklisted */
> +			if (!tev->point.symbol && !pev->uprobes)
> +				continue;
> +
> +			/* Set new name for tev (and update namelist) */
> +			ret = probe_trace_event__set_name(tev, pev,
> +							  namelist, true);
> +			if (ret)
> +				break;
> +
> +			if (!cur_name || strcmp(cur_name, tev->event)) {
> +				printf("%sftrace.event.kprobes.%s.probe = ",

Oops, I must not sleep well. this must be "%s.probes = ".
And I found this forgot to support return probes too.
Let me update it...

Thanks,


-- 
Masami Hiramatsu <mhiramat@kernel.org>

[PATCH 0/3] perf probe: Boot time tracing support followup

[Masami Hiramatsu]

Hi Arnaldo,

I found that I made 2 mistakes at the last series, typo and the return probe.
Since bootconfig have to use %return style definition for the return
probes, I have to update the synthesize_kprobe_trace_def() too.

So here are the follow up patches which update the last patch in the previous
series, and add the return probe support and documentation update.


Thank you,

---

Masami Hiramatsu (3):
      [v2] perf/probe: Add --bootconfig to output definition in bootconfig format
      perf/probe: Show return probe correctly with --bootconfig
      perf/probe: docs: Add --bootconfig option to perf-probe manual


 tools/perf/Documentation/perf-probe.txt |   15 +++++-
 tools/perf/builtin-probe.c              |   12 ++++-
 tools/perf/util/probe-event.c           |   82 +++++++++++++++++++++++++++++--
 tools/perf/util/probe-event.h           |    2 +
 4 files changed, 105 insertions(+), 6 deletions(-)

--
Masami Hiramatsu (Linaro) <mhiramat@kernel.org>

[PATCH 1/3] [v2] perf/probe: Add --bootconfig to output definition in bootconfig format

[Masami Hiramatsu]

Now the boot-time tracing supports the kprobes events and that
must be written in bootconfig file as following format.

ftrace.event.kprobes.<EVENT_NAME>.probes = <PROBE-DEF>

The perf probe already supports --definition (-D) action to
show the probe definitions, but the format is for the tracefs.

[p|r][:EVENT_NAME] <PROBE-DEF>

This adds --bootconfig option for -D action so that it output
the probe definitions in bootconfig fromat. E.g.

 $ perf probe --bootconfig -D "path_lookupat:7 err:s32 s:string"
 ftrace.event.kprobes.path_lookupat_L7.probes = 'path_lookupat.isra.0+309 err_s32=%ax:s32 s_string=+0(%r13):string'

Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org>
---
 Changes in v2
  - Fix boot time tracing option typo (probe -> probes).
---
 tools/perf/builtin-probe.c    |   12 ++++++-
 tools/perf/util/probe-event.c |   72 +++++++++++++++++++++++++++++++++++++++++
 tools/perf/util/probe-event.h |    2 +
 3 files changed, 85 insertions(+), 1 deletion(-)

diff --git a/tools/perf/builtin-probe.c b/tools/perf/builtin-probe.c
index 6b1507566770..2bfd41df621c 100644
--- a/tools/perf/builtin-probe.c
+++ b/tools/perf/builtin-probe.c
@@ -347,7 +347,10 @@ static int perf_add_probe_events(struct perf_probe_event *pevs, int npevs)
 		goto out_cleanup;
 
 	if (params.command == 'D') {	/* it shows definition */
-		ret = show_probe_trace_events(pevs, npevs);
+		if (probe_conf.bootconfig)
+			ret = show_bootconfig_events(pevs, npevs);
+		else
+			ret = show_probe_trace_events(pevs, npevs);
 		goto out_cleanup;
 	}
 
@@ -581,6 +584,8 @@ __cmd_probe(int argc, const char **argv)
 		   "Look for files with symbols relative to this directory"),
 	OPT_CALLBACK(0, "target-ns", NULL, "pid",
 		     "target pid for namespace contexts", opt_set_target_ns),
+	OPT_BOOLEAN(0, "bootconfig", &probe_conf.bootconfig,
+		    "Output probe definition with bootconfig format"),
 	OPT_END()
 	};
 	int ret;
@@ -692,6 +697,11 @@ __cmd_probe(int argc, const char **argv)
 		}
 		break;
 	case 'D':
+		if (probe_conf.bootconfig && params.uprobes) {
+			pr_err("  Error: --bootconfig doesn't support uprobes.\n");
+			return -EINVAL;
+		}
+		__fallthrough;
 	case 'a':
 
 		/* Ensure the last given target is used */
diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c
index 505c0702dbe2..8e30bf97cf0f 100644
--- a/tools/perf/util/probe-event.c
+++ b/tools/perf/util/probe-event.c
@@ -3564,6 +3564,78 @@ int show_probe_trace_events(struct perf_probe_event *pevs, int npevs)
 	return ret;
 }
 
+static int show_bootconfig_event(struct probe_trace_event *tev)
+{
+	struct probe_trace_point *tp = &tev->point;
+	struct strbuf buf;
+	char *ret = NULL;
+	int err;
+
+	if (strbuf_init(&buf, 32) < 0)
+		return -ENOMEM;
+
+	err = synthesize_kprobe_trace_def(tp, &buf);
+	if (err >= 0)
+		err = synthesize_probe_trace_args(tev, &buf);
+	if (err >= 0)
+		ret = strbuf_detach(&buf, NULL);
+	strbuf_release(&buf);
+
+	if (ret) {
+		printf("'%s'", ret);
+		free(ret);
+	}
+
+	return err;
+}
+
+int show_bootconfig_events(struct perf_probe_event *pevs, int npevs)
+{
+	struct strlist *namelist = strlist__new(NULL, NULL);
+	struct probe_trace_event *tev;
+	struct perf_probe_event *pev;
+	char *cur_name = NULL;
+	int i, j, ret = 0;
+
+	if (!namelist)
+		return -ENOMEM;
+
+	for (j = 0; j < npevs && !ret; j++) {
+		pev = &pevs[j];
+		if (pev->group && strcmp(pev->group, "probe"))
+			pr_warning("WARN: Group name %s is ignored\n", pev->group);
+		if (pev->uprobes) {
+			pr_warning("ERROR: Bootconfig doesn't support uprobes\n");
+			ret = -EINVAL;
+			break;
+		}
+		for (i = 0; i < pev->ntevs && !ret; i++) {
+			tev = &pev->tevs[i];
+			/* Skip if the symbol is out of .text or blacklisted */
+			if (!tev->point.symbol && !pev->uprobes)
+				continue;
+
+			/* Set new name for tev (and update namelist) */
+			ret = probe_trace_event__set_name(tev, pev,
+							  namelist, true);
+			if (ret)
+				break;
+
+			if (!cur_name || strcmp(cur_name, tev->event)) {
+				printf("%sftrace.event.kprobes.%s.probes = ",
+					cur_name ? "\n" : "", tev->event);
+				cur_name = tev->event;
+			} else
+				printf(", ");
+			ret = show_bootconfig_event(tev);
+		}
+	}
+	printf("\n");
+	strlist__delete(namelist);
+
+	return ret;
+}
+
 int apply_perf_probe_events(struct perf_probe_event *pevs, int npevs)
 {
 	int i, ret = 0;
diff --git a/tools/perf/util/probe-event.h b/tools/perf/util/probe-event.h
index 4f0eb3a20c36..65769d7949a3 100644
--- a/tools/perf/util/probe-event.h
+++ b/tools/perf/util/probe-event.h
@@ -15,6 +15,7 @@ struct probe_conf {
 	bool	force_add;
 	bool	no_inlines;
 	bool	cache;
+	bool	bootconfig;
 	int	max_probes;
 	unsigned long	magic_num;
 };
@@ -163,6 +164,7 @@ int add_perf_probe_events(struct perf_probe_event *pevs, int npevs);
 int convert_perf_probe_events(struct perf_probe_event *pevs, int npevs);
 int apply_perf_probe_events(struct perf_probe_event *pevs, int npevs);
 int show_probe_trace_events(struct perf_probe_event *pevs, int npevs);
+int show_bootconfig_events(struct perf_probe_event *pevs, int npevs);
 void cleanup_perf_probe_events(struct perf_probe_event *pevs, int npevs);
 
 struct strfilter;

[PATCH 2/3] perf/probe: Show return probe correctly with --bootconfig

[Masami Hiramatsu]

Since the bootconfig has to use "%return" suffix for the return
probes, perf probe also needs to add that suffix if the user
defines the return probe. E.g.

  $ perf probe -k vmlinux --bootconfig -D vfs_read%return
  ftrace.event.kprobes.vfs_read__return.probe = 'vfs_read+0%return'

Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org>
---
 tools/perf/util/probe-event.c |   12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c
index 8e30bf97cf0f..99624e39314b 100644
--- a/tools/perf/util/probe-event.c
+++ b/tools/perf/util/probe-event.c
@@ -2166,15 +2166,17 @@ synthesize_uprobe_trace_def(struct probe_trace_point *tp, struct strbuf *buf)
 }
 
 static int
-synthesize_kprobe_trace_def(struct probe_trace_point *tp, struct strbuf *buf)
+synthesize_kprobe_trace_def(struct probe_trace_point *tp, struct strbuf *buf,
+			    bool show_return)
 {
 	if (!strncmp(tp->symbol, "0x", 2)) {
 		/* Absolute address. See try_to_find_absolute_address() */
 		return strbuf_addf(buf, "%s%s0x%lx", tp->module ?: "",
 				  tp->module ? ":" : "", tp->address);
 	} else {
-		return strbuf_addf(buf, "%s%s%s+%lu", tp->module ?: "",
-				tp->module ? ":" : "", tp->symbol, tp->offset);
+		return strbuf_addf(buf, "%s%s%s+%lu%s", tp->module ?: "",
+				tp->module ? ":" : "", tp->symbol, tp->offset,
+				show_return && tp->retprobe ? "%return" : "");
 	}
 }
 
@@ -2195,7 +2197,7 @@ char *synthesize_probe_trace_command(struct probe_trace_event *tev)
 	if (tev->uprobes)
 		err = synthesize_uprobe_trace_def(tp, &buf);
 	else
-		err = synthesize_kprobe_trace_def(tp, &buf);
+		err = synthesize_kprobe_trace_def(tp, &buf, false);
 
 	if (err >= 0)
 		err = synthesize_probe_trace_args(tev, &buf);
@@ -3574,7 +3576,7 @@ static int show_bootconfig_event(struct probe_trace_event *tev)
 	if (strbuf_init(&buf, 32) < 0)
 		return -ENOMEM;
 
-	err = synthesize_kprobe_trace_def(tp, &buf);
+	err = synthesize_kprobe_trace_def(tp, &buf, true);
 	if (err >= 0)
 		err = synthesize_probe_trace_args(tev, &buf);
 	if (err >= 0)

[PATCH 3/3] perf/probe: docs: Add --bootconfig option to perf-probe manual

[Masami Hiramatsu]

Add --bootconfig option and example to the perf-probe manual.

Signed-off-by: Masami Hiramatsu <mhiramat@kernel.org>
---
 tools/perf/Documentation/perf-probe.txt |   15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/tools/perf/Documentation/perf-probe.txt b/tools/perf/Documentation/perf-probe.txt
index ed3ecfa422e1..d0908170c3d2 100644
--- a/tools/perf/Documentation/perf-probe.txt
+++ b/tools/perf/Documentation/perf-probe.txt
@@ -103,7 +103,14 @@ OPTIONS
 -D::
 --definition=::
 	Show trace-event definition converted from given probe-event instead
-	of write it into tracing/[k,u]probe_events.
+	of write it into tracing/[k,u]probe_events. This accepts --bootconfig
+	option.
+
+--bootconfig::
+	Show the trace-event definition in Extra Boot Configuration (bootconfig)
+	syntax. This helps users to write the probe event definitions for the
+	boot-time tracing.
+	Note that this must be used with --definition(-D) option.
 
 --filter=FILTER::
 	(Only for --vars and --funcs) Set filter. FILTER is a combination of glob
@@ -294,6 +301,12 @@ Add a probe in a source file using special characters by backslash escape
 
  ./perf probe -x /opt/test/a.out 'foo\+bar.c:4'
 
+Define a kprobe event on the line 7 of path_lookup for the boot-time tracing
+and set it to the initrd.img.
+
+ ./perf probe -k /boot/vmlinux --bootconfig -D "path_lookup:7" > tmp.bconf
+ bootconfig -a tmp.bconf /boot/initrd.img
+
 
 SEE ALSO
 --------