[PATCH 00/19] perf: ftrace enhancement

[PATCH 00/19] perf: ftrace enhancement

[Changbin Du]

The perf has basic kernel ftrace support but lack support of most tracing
options. This serias is target to enhance the perf ftrace functionality so
that we can make full use of kernel ftrace with only perf.

In general, this serias be cataloged into two main changes:
  1) Improve usability of existing functions. For example, we don't need to type
     extra option to select the tracer.
  2) Add new options to support all other ftrace functions.

Here is a glance of all ftrace functions with this serias:
  * - improved existing options.
  + - new added options.

$ sudo perf ftrace -h

 Usage: perf ftrace [<options>] [<command>]
    or: perf ftrace [<options>] -- <command> [<options>]

*   -a, --all-cpus        system-wide collection from all CPUs
+   -b, --buffer-size <n>
                          size of per cpu buffer in kb
    -C, --cpu <cpu>       list of cpus to monitor
+   -d, --delay <n>       Wait <n> ms before tracing
    -D, --graph-depth <n>
                          Max depth for function graph tracer
*   -G, --graph-funcs <func>
                          Set graph filter on given functions (imply to use function_graph tracer)
    -g, --nograph-funcs <func>
                          Set nograph filter on given functions (imply to use function_graph tracer)
+   -L, --list-functions  List available functions to filter
+   -l, --long-info       Show process names, PIDs, timestamps, irq-info if available
    -N, --notrace-funcs <func>
                          do not trace given functions
+   -P, --no-pager        Do not use pager
    -p, --pid <pid>       trace on existing process id
+   -s, --func-stack-trace
                          Show kernel stack trace for function tracer
+   -t, --tid <tid>       trace on existing thread id (exclusive to --pid)
    -T, --trace-funcs <func>
                          trace given functions only
+   -u, --userstacktrace  Show stacktrace of the current user space thread
    -v, --verbose         be more verbose
+       --funcgraph-tail  Show function tails comment (function_graph only)
+       --latency-format  displays additional information about the latency (function_graph only)
+       --nofuncgraph-irqs
                          Ignore functions that happen inside interrupt (function_graph only)
+       --nosleep-time    Measure on-CPU time only (function_graph only)
+       --trace-children  Trace children processes
+       --tracing-thresh <n>
                          Only show functions of which the duration is greater than <n>µs


Changbin Du (19):
  perf ftrace: trace system wide if no target is given
  perf ftrace: detect workload failure
  perf ftrace: select function/function_graph tracer automatically
  perf ftrace: add support for tracing option 'func_stack_trace'
  perf ftrace: add option '-l/--list-functions' to list available
    functions
  perf ftrace: add support for trace option sleep-time
  perf ftrace: add support for trace option funcgraph-irqs
  perf ftrace: add option -l/--long-info to show more info
  perf ftrace: add support for trace option tracing_thresh
  perf ftrace: add support for trace option funcgraph-tail
  perf ftrace: add option '-u/--userstacktrace' to show userspace
    stacktrace
  perf ftrace: add support for tracing children processes
  perf ftrace: add option '-b/--buffer-size' to set per-cpu buffer size
  perf ftrace: add option -P/--no-pager to disable pager
  perf ftrace: show trace column header
  perf ftrace: add option -t/--tid to filter by thread id
  perf ftrace: add option -d/--delay to delay tracing
  perf ftrace: add option --latency-format to display more info about
    delay
  perf ftrace: add change log

 tools/perf/Documentation/perf-config.txt |   5 -
 tools/perf/builtin-ftrace.c              | 388 ++++++++++++++++++++---
 2 files changed, 345 insertions(+), 48 deletions(-)

-- 
2.25.1

[PATCH 01/19] perf ftrace: trace system wide if no target is given

[Changbin Du]

This align ftrace to other perf sub-commands that if no target specified
then we trace all functions.

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index d5adc417a4ca..11fc02037899 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -493,7 +493,7 @@ int cmd_ftrace(int argc, const char **argv)
 	argc = parse_options(argc, argv, ftrace_options, ftrace_usage,
 			    PARSE_OPT_STOP_AT_NON_OPTION);
 	if (!argc && target__none(&ftrace.target))
-		usage_with_options(ftrace_usage, ftrace_options);
+		ftrace.target.system_wide = true;
 
 	ret = target__validate(&ftrace.target);
 	if (ret) {
-- 
2.25.1

[PATCH 02/19] perf ftrace: detect workload failure

[Changbin Du]

Currently there's no error message prompted if we failed to start workload.
And we still get some trace which is confusing. Let's tell users what
happened.

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index 11fc02037899..5584f8dec25d 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -45,6 +45,7 @@ struct filter_entry {
 	char			name[];
 };
 
+static volatile int workload_exec_errno;
 static bool done;
 
 static void sig_handler(int sig __maybe_unused)
@@ -63,7 +64,7 @@ static void ftrace__workload_exec_failed_signal(int signo __maybe_unused,
 						siginfo_t *info __maybe_unused,
 						void *ucontext __maybe_unused)
 {
-	/* workload_exec_errno = info->si_value.sival_int; */
+	workload_exec_errno = info->si_value.sival_int;
 	done = true;
 }
 
@@ -382,6 +383,14 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 
 	write_tracing_file("tracing_on", "0");
 
+	if (workload_exec_errno) {
+		const char *emsg = str_error_r(workload_exec_errno, buf, sizeof(buf));
+		/* flush stdout first so below error msg appears at the end. */
+		fflush(stdout);
+		pr_err("workload failed: %s\n", emsg);
+		goto out_close_fd;
+	}
+
 	/* read remaining buffer contents */
 	while (true) {
 		int n = read(trace_fd, buf, sizeof(buf));
@@ -396,7 +405,7 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 out_reset:
 	reset_tracing_files(ftrace);
 out:
-	return done ? 0 : -1;
+	return (done && !workload_exec_errno) ? 0 : -1;
 }
 
 static int perf_ftrace_config(const char *var, const char *value, void *cb)
-- 
2.25.1

[PATCH 03/19] perf ftrace: select function/function_graph tracer automatically

[Changbin Du]

The '-g/-G' options have already implied function_graph tracer should be
used instead of function tracer. So the extra option '--tracer' can be
killed.

This patch changes the behavior as below:
  - By default, function tracer is used.
  - If '-g' or '-G' option is on, then function_graph tracer is used.
  - The perf configuration item 'ftrace.tracer' is removed.
  - The default filter for -G is to trace all functions.

Here are some examples.

This will start tracing all functions using function tracer:
  $ sudo perf ftrace

This will trace all functions using function graph tracer:
  $ sudo perf ftrace -G

This will trace function vfs_read using function graph tracer:
  $ sudo perf ftrace -G vfs_read

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/Documentation/perf-config.txt |  5 ---
 tools/perf/builtin-ftrace.c              | 39 ++++++------------------
 2 files changed, 9 insertions(+), 35 deletions(-)

diff --git a/tools/perf/Documentation/perf-config.txt b/tools/perf/Documentation/perf-config.txt
index f16d8a71d3f5..fad6c48ed76a 100644
--- a/tools/perf/Documentation/perf-config.txt
+++ b/tools/perf/Documentation/perf-config.txt
@@ -612,11 +612,6 @@ trace.*::
 		"libbeauty", the default, to use the same argument beautifiers used in the
 		strace-like sys_enter+sys_exit lines.
 
-ftrace.*::
-	ftrace.tracer::
-		Can be used to select the default tracer. Possible values are
-		'function' and 'function_graph'.
-
 llvm.*::
 	llvm.clang-path::
 		Path to clang. If omit, search it from $PATH.
diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index 5584f8dec25d..57e656c35d28 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -27,7 +27,7 @@
 #include "util/cap.h"
 #include "util/config.h"
 
-#define DEFAULT_TRACER  "function_graph"
+#define DEFAULT_TRACER  "function"
 
 struct perf_ftrace {
 	struct evlist		*evlist;
@@ -408,26 +408,6 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 	return (done && !workload_exec_errno) ? 0 : -1;
 }
 
-static int perf_ftrace_config(const char *var, const char *value, void *cb)
-{
-	struct perf_ftrace *ftrace = cb;
-
-	if (!strstarts(var, "ftrace."))
-		return 0;
-
-	if (strcmp(var, "ftrace.tracer"))
-		return -1;
-
-	if (!strcmp(value, "function_graph") ||
-	    !strcmp(value, "function")) {
-		ftrace->tracer = value;
-		return 0;
-	}
-
-	pr_err("Please select \"function_graph\" (default) or \"function\"\n");
-	return -1;
-}
-
 static int parse_filter_func(const struct option *opt, const char *str,
 			     int unset __maybe_unused)
 {
@@ -467,8 +447,6 @@ int cmd_ftrace(int argc, const char **argv)
 		NULL
 	};
 	const struct option ftrace_options[] = {
-	OPT_STRING('t', "tracer", &ftrace.tracer, "tracer",
-		   "tracer to use: function_graph(default) or function"),
 	OPT_STRING('p', "pid", &ftrace.target.pid, "pid",
 		   "trace on existing process id"),
 	OPT_INCR('v', "verbose", &verbose,
@@ -481,10 +459,12 @@ int cmd_ftrace(int argc, const char **argv)
 		     "trace given functions only", parse_filter_func),
 	OPT_CALLBACK('N', "notrace-funcs", &ftrace.notrace, "func",
 		     "do not trace given functions", parse_filter_func),
-	OPT_CALLBACK('G', "graph-funcs", &ftrace.graph_funcs, "func",
-		     "Set graph filter on given functions", parse_filter_func),
+	OPT_CALLBACK_DEFAULT('G', "graph-funcs", &ftrace.graph_funcs, "func",
+		     "Set graph filter on given functions (imply to use function_graph tracer)",
+		     parse_filter_func, "*"),
 	OPT_CALLBACK('g', "nograph-funcs", &ftrace.nograph_funcs, "func",
-		     "Set nograph filter on given functions", parse_filter_func),
+		     "Set nograph filter on given functions (imply to use function_graph tracer)",
+		     parse_filter_func),
 	OPT_INTEGER('D', "graph-depth", &ftrace.graph_depth,
 		    "Max depth for function graph tracer"),
 	OPT_END()
@@ -495,15 +475,14 @@ int cmd_ftrace(int argc, const char **argv)
 	INIT_LIST_HEAD(&ftrace.graph_funcs);
 	INIT_LIST_HEAD(&ftrace.nograph_funcs);
 
-	ret = perf_config(perf_ftrace_config, &ftrace);
-	if (ret < 0)
-		return -1;
-
 	argc = parse_options(argc, argv, ftrace_options, ftrace_usage,
 			    PARSE_OPT_STOP_AT_NON_OPTION);
 	if (!argc && target__none(&ftrace.target))
 		ftrace.target.system_wide = true;
 
+	if (!list_empty(&ftrace.graph_funcs) || !list_empty(&ftrace.nograph_funcs))
+		ftrace.tracer = "function_graph";
+
 	ret = target__validate(&ftrace.target);
 	if (ret) {
 		char errbuf[512];
-- 
2.25.1

[PATCH 04/19] perf ftrace: add support for tracing option 'func_stack_trace'

[Changbin Du]

This adds support to display call trace for function tracer. To do this,
just specify a '-s' option.

$ sudo perf ftrace -T vfs_read -s
 iio-sensor-prox-855   [003]   6168.369657: vfs_read <-ksys_read
 iio-sensor-prox-855   [003]   6168.369677: <stack trace>
 => vfs_read
 => ksys_read
 => __x64_sys_read
 => do_syscall_64
 => entry_SYSCALL_64_after_hwframe
 ...

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 38 +++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index 57e656c35d28..1d30c2d5f88b 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -38,6 +38,7 @@ struct perf_ftrace {
 	struct list_head	graph_funcs;
 	struct list_head	nograph_funcs;
 	int			graph_depth;
+	bool			func_stack_trace;
 };
 
 struct filter_entry {
@@ -128,9 +129,27 @@ static int append_tracing_file(const char *name, const char *val)
 	return __write_tracing_file(name, val, true);
 }
 
+static int write_tracing_option_file(const char *name, const char *val)
+{
+	char *file;
+	int ret;
+
+	if (asprintf(&file, "options/%s", name) < 0)
+		return -1;
+
+	ret = __write_tracing_file(file, val, false);
+	free(file);
+	return ret;
+}
+
 static int reset_tracing_cpu(void);
 static void reset_tracing_filters(void);
 
+static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
+{
+	write_tracing_option_file("func_stack_trace", "0");
+}
+
 static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
 {
 	if (write_tracing_file("tracing_on", "0") < 0)
@@ -149,6 +168,7 @@ static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
 		return -1;
 
 	reset_tracing_filters();
+	reset_tracing_options(ftrace);
 	return 0;
 }
 
@@ -204,6 +224,17 @@ static int set_tracing_cpu(struct perf_ftrace *ftrace)
 	return set_tracing_cpumask(cpumap);
 }
 
+static int set_tracing_func_stack_trace(struct perf_ftrace *ftrace)
+{
+	if (!ftrace->func_stack_trace)
+		return 0;
+
+	if (write_tracing_option_file("func_stack_trace", "1") < 0)
+		return -1;
+
+	return 0;
+}
+
 static int reset_tracing_cpu(void)
 {
 	struct perf_cpu_map *cpumap = perf_cpu_map__new(NULL);
@@ -326,6 +357,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 		goto out_reset;
 	}
 
+	if (set_tracing_func_stack_trace(ftrace) < 0) {
+		pr_err("failed to set tracing option func_stack_trace\n");
+		goto out_reset;
+	}
+
 	if (set_tracing_filters(ftrace) < 0) {
 		pr_err("failed to set tracing filters\n");
 		goto out_reset;
@@ -459,6 +495,8 @@ int cmd_ftrace(int argc, const char **argv)
 		     "trace given functions only", parse_filter_func),
 	OPT_CALLBACK('N', "notrace-funcs", &ftrace.notrace, "func",
 		     "do not trace given functions", parse_filter_func),
+	OPT_BOOLEAN('s', "func-stack-trace", &ftrace.func_stack_trace,
+		    "Show kernel stack trace for function tracer"),
 	OPT_CALLBACK_DEFAULT('G', "graph-funcs", &ftrace.graph_funcs, "func",
 		     "Set graph filter on given functions (imply to use function_graph tracer)",
 		     parse_filter_func, "*"),
-- 
2.25.1

[PATCH 05/19] perf ftrace: add option '-l/--list-functions' to list available functions

[Changbin Du]

This adds an option '-l/--list-functions' to list all available functions
which is read from tracing file 'available_filter_functions'.

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 43 +++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index 1d30c2d5f88b..8133d910d5d8 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -33,6 +33,7 @@ struct perf_ftrace {
 	struct evlist		*evlist;
 	struct target		target;
 	const char		*tracer;
+	bool			list_avail_functions;
 	struct list_head	filters;
 	struct list_head	notrace;
 	struct list_head	graph_funcs;
@@ -142,6 +143,43 @@ static int write_tracing_option_file(const char *name, const char *val)
 	return ret;
 }
 
+static int read_tracing_file_to_stdout(const char *name)
+{
+	char buf[4096];
+	char *file;
+	int fd;
+	int ret = -1;
+
+	file = get_tracing_file(name);
+	if (!file) {
+		pr_debug("cannot get tracing file: %s\n", name);
+		return -1;
+	}
+
+	fd = open(file, O_RDONLY);
+	if (fd < 0) {
+		pr_debug("cannot open tracing file: %s: %s\n",
+			 name, str_error_r(errno, buf, sizeof(buf)));
+		goto out;
+	}
+
+	/* read contents to stdout */
+	while (true) {
+		int n = read(fd, buf, sizeof(buf));
+		if (n <= 0)
+			goto out_close;
+		if (fwrite(buf, n, 1, stdout) != 1)
+			goto out_close;
+	}
+	ret = 0;
+
+out_close:
+	close(fd);
+out:
+	put_tracing_file(file);
+	return ret;
+}
+
 static int reset_tracing_cpu(void);
 static void reset_tracing_filters(void);
 
@@ -332,6 +370,9 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 	signal(SIGCHLD, sig_handler);
 	signal(SIGPIPE, sig_handler);
 
+	if (ftrace->list_avail_functions)
+		return read_tracing_file_to_stdout("available_filter_functions");
+
 	if (reset_tracing_files(ftrace) < 0) {
 		pr_err("failed to reset ftrace\n");
 		goto out;
@@ -483,6 +524,8 @@ int cmd_ftrace(int argc, const char **argv)
 		NULL
 	};
 	const struct option ftrace_options[] = {
+	OPT_BOOLEAN('L', "list-functions", &ftrace.list_avail_functions,
+		    "List available functions to filter"),
 	OPT_STRING('p', "pid", &ftrace.target.pid, "pid",
 		   "trace on existing process id"),
 	OPT_INCR('v', "verbose", &verbose,
-- 
2.25.1

[PATCH 06/19] perf ftrace: add support for trace option sleep-time

[Changbin Du]

This adds an option '--nosleep-time' which allow us only to measure
on-CPU time. This option is function_graph tracer only.

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index 8133d910d5d8..d3fcf3b0b792 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -40,6 +40,7 @@ struct perf_ftrace {
 	struct list_head	nograph_funcs;
 	int			graph_depth;
 	bool			func_stack_trace;
+	bool			nosleep_time;
 };
 
 struct filter_entry {
@@ -186,6 +187,7 @@ static void reset_tracing_filters(void);
 static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
 {
 	write_tracing_option_file("func_stack_trace", "0");
+	write_tracing_option_file("sleep-time", "1");
 }
 
 static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
@@ -345,6 +347,17 @@ static int set_tracing_depth(struct perf_ftrace *ftrace)
 	return 0;
 }
 
+static int set_tracing_sleep_time(struct perf_ftrace *ftrace)
+{
+	if (!ftrace->nosleep_time)
+		return 0;
+
+	if (write_tracing_option_file("sleep-time", "0") < 0)
+		return -1;
+
+	return 0;
+}
+
 static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 {
 	char *trace_file;
@@ -413,6 +426,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 		goto out_reset;
 	}
 
+	if (set_tracing_sleep_time(ftrace) < 0) {
+		pr_err("failed to set tracing option sleep-time\n");
+		goto out_reset;
+	}
+
 	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
 		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
 		goto out_reset;
@@ -548,6 +566,8 @@ int cmd_ftrace(int argc, const char **argv)
 		     parse_filter_func),
 	OPT_INTEGER('D', "graph-depth", &ftrace.graph_depth,
 		    "Max depth for function graph tracer"),
+	OPT_BOOLEAN(0, "nosleep-time", &ftrace.nosleep_time,
+		    "Measure on-CPU time only (function_graph only)"),
 	OPT_END()
 	};
 
-- 
2.25.1

[PATCH 07/19] perf ftrace: add support for trace option funcgraph-irqs

[Changbin Du]

This adds an option '--nofuncgraph-irqs' to filter out functions executed
in irq context.

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index d3fcf3b0b792..b16600a16efa 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -41,6 +41,7 @@ struct perf_ftrace {
 	int			graph_depth;
 	bool			func_stack_trace;
 	bool			nosleep_time;
+	bool			nofuncgraph_irqs;
 };
 
 struct filter_entry {
@@ -188,6 +189,7 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
 {
 	write_tracing_option_file("func_stack_trace", "0");
 	write_tracing_option_file("sleep-time", "1");
+	write_tracing_option_file("funcgraph-irqs", "1");
 }
 
 static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
@@ -358,6 +360,17 @@ static int set_tracing_sleep_time(struct perf_ftrace *ftrace)
 	return 0;
 }
 
+static int set_tracing_funcgraph_irqs(struct perf_ftrace *ftrace)
+{
+	if (!ftrace->nofuncgraph_irqs)
+		return 0;
+
+	if (write_tracing_option_file("funcgraph-irqs", "0") < 0)
+		return -1;
+
+	return 0;
+}
+
 static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 {
 	char *trace_file;
@@ -431,6 +444,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 		goto out_reset;
 	}
 
+	if (set_tracing_funcgraph_irqs(ftrace) < 0) {
+		pr_err("failed to set tracing option funcgraph-irqs\n");
+		goto out_reset;
+	}
+
 	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
 		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
 		goto out_reset;
@@ -568,6 +586,8 @@ int cmd_ftrace(int argc, const char **argv)
 		    "Max depth for function graph tracer"),
 	OPT_BOOLEAN(0, "nosleep-time", &ftrace.nosleep_time,
 		    "Measure on-CPU time only (function_graph only)"),
+	OPT_BOOLEAN(0, "nofuncgraph-irqs", &ftrace.nofuncgraph_irqs,
+		    "Ignore functions that happen inside interrupt (function_graph only)"),
 	OPT_END()
 	};
 
-- 
2.25.1

[PATCH 08/19] perf ftrace: add option -l/--long-info to show more info

[Changbin Du]

Sometimes we want ftrace display more and longer information about trace.

$ sudo perf ftrace -G -l
 6800.190937 |   4)   <...>-7683   |   2.072 us    |  mutex_unlock();
 6800.190941 |   4)   <...>-7683   |   2.171 us    |  __fsnotify_parent();
 6800.190943 |   4)   <...>-7683   |   1.497 us    |  fsnotify();
 6800.190944 |   4)   <...>-7683   |   0.775 us    |  __sb_end_write();
 6800.190945 |   4)   <...>-7683   |   0.854 us    |  fpregs_assert_state_consistent();
 6800.190947 |   4)   <...>-7683   |               |  do_syscall_64() {
 6800.190948 |   4)   <...>-7683   |               |    __x64_sys_close() {
 6800.190948 |   4)   <...>-7683   |               |      __close_fd() {
 6800.190948 |   4)   <...>-7683   |   0.322 us    |        _raw_spin_lock();
 6800.190949 |   4)   <...>-7683   |               |        filp_close() {
 6800.190949 |   4)   <...>-7683   |   0.320 us    |          dnotify_flush();
 6800.190950 |   4)   <...>-7683   |   0.325 us    |          locks_remove_posix();

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index b16600a16efa..f11f2d3431b0 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -42,6 +42,7 @@ struct perf_ftrace {
 	bool			func_stack_trace;
 	bool			nosleep_time;
 	bool			nofuncgraph_irqs;
+	bool			long_info;
 };
 
 struct filter_entry {
@@ -190,6 +191,9 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
 	write_tracing_option_file("func_stack_trace", "0");
 	write_tracing_option_file("sleep-time", "1");
 	write_tracing_option_file("funcgraph-irqs", "1");
+	write_tracing_option_file("funcgraph-proc", "0");
+	write_tracing_option_file("funcgraph-abstime", "0");
+	write_tracing_option_file("irq-info", "0");
 }
 
 static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
@@ -371,6 +375,23 @@ static int set_tracing_funcgraph_irqs(struct perf_ftrace *ftrace)
 	return 0;
 }
 
+static int set_tracing_long_info(struct perf_ftrace *ftrace)
+{
+	if (!ftrace->long_info)
+		return 0;
+
+	if (write_tracing_option_file("funcgraph-proc", "1") < 0)
+		return -1;
+
+	if (write_tracing_option_file("funcgraph-abstime", "1") < 0)
+		return -1;
+
+	if (write_tracing_option_file("irq-info", "1") < 0)
+		return -1;
+
+	return 0;
+}
+
 static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 {
 	char *trace_file;
@@ -449,6 +470,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 		goto out_reset;
 	}
 
+	if (set_tracing_long_info(ftrace) < 0) {
+		pr_err("failed to set tracing option funcgraph-proc/funcgraph-abstime/irq-info\n");
+		goto out_reset;
+	}
+
 	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
 		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
 		goto out_reset;
@@ -588,6 +614,8 @@ int cmd_ftrace(int argc, const char **argv)
 		    "Measure on-CPU time only (function_graph only)"),
 	OPT_BOOLEAN(0, "nofuncgraph-irqs", &ftrace.nofuncgraph_irqs,
 		    "Ignore functions that happen inside interrupt (function_graph only)"),
+	OPT_BOOLEAN('l', "long-info", &ftrace.long_info,
+		    "Show process names, PIDs, timestamps, irq-info if available"),
 	OPT_END()
 	};
 
-- 
2.25.1

[PATCH 09/19] perf ftrace: add support for trace option tracing_thresh

[Changbin Du]

This adds an option '--tracing-thresh' to setup trace duration threshold
for funcgraph tracer.

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index f11f2d3431b0..20bc14d6c5fb 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -43,6 +43,7 @@ struct perf_ftrace {
 	bool			nosleep_time;
 	bool			nofuncgraph_irqs;
 	bool			long_info;
+	unsigned		tracing_thresh;
 };
 
 struct filter_entry {
@@ -213,6 +214,9 @@ static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
 	if (write_tracing_file("max_graph_depth", "0") < 0)
 		return -1;
 
+	if (write_tracing_file("tracing_thresh", "0") < 0)
+		return -1;
+
 	reset_tracing_filters();
 	reset_tracing_options(ftrace);
 	return 0;
@@ -392,6 +396,21 @@ static int set_tracing_long_info(struct perf_ftrace *ftrace)
 	return 0;
 }
 
+static int set_tracing_thresh(struct perf_ftrace *ftrace)
+{
+	char buf[16];
+
+	if (ftrace->tracing_thresh == 0)
+		return 0;
+
+	snprintf(buf, sizeof(buf), "%d", ftrace->tracing_thresh);
+
+	if (write_tracing_file("tracing_thresh", buf) < 0)
+		return -1;
+
+	return 0;
+}
+
 static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 {
 	char *trace_file;
@@ -475,6 +494,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 		goto out_reset;
 	}
 
+	if (set_tracing_thresh(ftrace) < 0) {
+		pr_err("failed to set tracing thresh\n");
+		goto out_reset;
+	}
+
 	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
 		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
 		goto out_reset;
@@ -616,6 +640,8 @@ int cmd_ftrace(int argc, const char **argv)
 		    "Ignore functions that happen inside interrupt (function_graph only)"),
 	OPT_BOOLEAN('l', "long-info", &ftrace.long_info,
 		    "Show process names, PIDs, timestamps, irq-info if available"),
+	OPT_UINTEGER(0, "tracing-thresh", &ftrace.tracing_thresh,
+		     "Only show functions of which the duration is greater than <n>µs"),
 	OPT_END()
 	};
 
-- 
2.25.1

[PATCH 10/19] perf ftrace: add support for trace option funcgraph-tail

[Changbin Du]

This adds an option '--funcgraph-tail' for function graph tracer.

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index 20bc14d6c5fb..2ef5d1c4b23c 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -42,6 +42,7 @@ struct perf_ftrace {
 	bool			func_stack_trace;
 	bool			nosleep_time;
 	bool			nofuncgraph_irqs;
+	bool			funcgraph_tail;
 	bool			long_info;
 	unsigned		tracing_thresh;
 };
@@ -192,6 +193,7 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
 	write_tracing_option_file("func_stack_trace", "0");
 	write_tracing_option_file("sleep-time", "1");
 	write_tracing_option_file("funcgraph-irqs", "1");
+	write_tracing_option_file("funcgraph-tail", "0");
 	write_tracing_option_file("funcgraph-proc", "0");
 	write_tracing_option_file("funcgraph-abstime", "0");
 	write_tracing_option_file("irq-info", "0");
@@ -411,6 +413,17 @@ static int set_tracing_thresh(struct perf_ftrace *ftrace)
 	return 0;
 }
 
+static int set_tracing_funcgraph_tail(struct perf_ftrace *ftrace)
+{
+	if (!ftrace->funcgraph_tail)
+		return 0;
+
+	if (write_tracing_option_file("funcgraph-tail", "1") < 0)
+		return -1;
+
+	return 0;
+}
+
 static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 {
 	char *trace_file;
@@ -499,6 +512,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 		goto out_reset;
 	}
 
+	if (set_tracing_funcgraph_tail(ftrace) < 0) {
+		pr_err("failed to set tracing option funcgraph-tail\n");
+		goto out_reset;
+	}
+
 	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
 		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
 		goto out_reset;
@@ -638,6 +656,8 @@ int cmd_ftrace(int argc, const char **argv)
 		    "Measure on-CPU time only (function_graph only)"),
 	OPT_BOOLEAN(0, "nofuncgraph-irqs", &ftrace.nofuncgraph_irqs,
 		    "Ignore functions that happen inside interrupt (function_graph only)"),
+	OPT_BOOLEAN(0, "funcgraph-tail", &ftrace.funcgraph_tail,
+		    "Show function tails comment (function_graph only)"),
 	OPT_BOOLEAN('l', "long-info", &ftrace.long_info,
 		    "Show process names, PIDs, timestamps, irq-info if available"),
 	OPT_UINTEGER(0, "tracing-thresh", &ftrace.tracing_thresh,
-- 
2.25.1

[PATCH 11/19] perf ftrace: add option '-u/--userstacktrace' to show userspace stacktrace

[Changbin Du]

This adds an option ''-u/--userstacktrace' for function tracer to display
userspace back trace.

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index 2ef5d1c4b23c..ab76ba66bd9e 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -40,6 +40,7 @@ struct perf_ftrace {
 	struct list_head	nograph_funcs;
 	int			graph_depth;
 	bool			func_stack_trace;
+	bool			userstacktrace;
 	bool			nosleep_time;
 	bool			nofuncgraph_irqs;
 	bool			funcgraph_tail;
@@ -197,6 +198,8 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
 	write_tracing_option_file("funcgraph-proc", "0");
 	write_tracing_option_file("funcgraph-abstime", "0");
 	write_tracing_option_file("irq-info", "0");
+	write_tracing_option_file("userstacktrace", "0");
+	write_tracing_option_file("sym-userobj", "0");
 }
 
 static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
@@ -287,6 +290,20 @@ static int set_tracing_func_stack_trace(struct perf_ftrace *ftrace)
 	return 0;
 }
 
+static int set_tracing_userstacktrace(struct perf_ftrace *ftrace)
+{
+	if (!ftrace->userstacktrace)
+		return 0;
+
+	if (write_tracing_option_file("userstacktrace", "1") < 0)
+		return -1;
+
+	if (write_tracing_option_file("sym-userobj", "1") < 0)
+		return -1;
+
+	return 0;
+}
+
 static int reset_tracing_cpu(void)
 {
 	struct perf_cpu_map *cpumap = perf_cpu_map__new(NULL);
@@ -482,6 +499,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 		goto out_reset;
 	}
 
+	if (set_tracing_userstacktrace(ftrace) < 0) {
+		pr_err("failed to set tracing option userstacktrace\n");
+		goto out_reset;
+	}
+
 	if (set_tracing_filters(ftrace) < 0) {
 		pr_err("failed to set tracing filters\n");
 		goto out_reset;
@@ -644,6 +666,8 @@ int cmd_ftrace(int argc, const char **argv)
 		     "do not trace given functions", parse_filter_func),
 	OPT_BOOLEAN('s', "func-stack-trace", &ftrace.func_stack_trace,
 		    "Show kernel stack trace for function tracer"),
+	OPT_BOOLEAN('u', "userstacktrace", &ftrace.userstacktrace,
+		    "Show stacktrace of the current user space thread"),
 	OPT_CALLBACK_DEFAULT('G', "graph-funcs", &ftrace.graph_funcs, "func",
 		     "Set graph filter on given functions (imply to use function_graph tracer)",
 		     parse_filter_func, "*"),
-- 
2.25.1

[PATCH 12/19] perf ftrace: add support for tracing children processes

[Changbin Du]

This adds an option '--trace-children' to allow us trace children
processes spawned by our target.

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index ab76ba66bd9e..8fd95c109fe8 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -46,6 +46,7 @@ struct perf_ftrace {
 	bool			funcgraph_tail;
 	bool			long_info;
 	unsigned		tracing_thresh;
+	bool			trace_children;
 };
 
 struct filter_entry {
@@ -200,6 +201,7 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
 	write_tracing_option_file("irq-info", "0");
 	write_tracing_option_file("userstacktrace", "0");
 	write_tracing_option_file("sym-userobj", "0");
+	write_tracing_option_file("function-fork", "0");
 }
 
 static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
@@ -441,6 +443,17 @@ static int set_tracing_funcgraph_tail(struct perf_ftrace *ftrace)
 	return 0;
 }
 
+static int set_tracing_trace_children(struct perf_ftrace *ftrace)
+{
+	if (!ftrace->trace_children)
+		return 0;
+
+	if (write_tracing_option_file("function-fork", "1") < 0)
+		return -1;
+
+	return 0;
+}
+
 static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 {
 	char *trace_file;
@@ -539,6 +552,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 		goto out_reset;
 	}
 
+	if (set_tracing_trace_children(ftrace) < 0) {
+		pr_err("failed to set tracing option function-fork\n");
+		goto out_reset;
+	}
+
 	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
 		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
 		goto out_reset;
@@ -686,6 +704,8 @@ int cmd_ftrace(int argc, const char **argv)
 		    "Show process names, PIDs, timestamps, irq-info if available"),
 	OPT_UINTEGER(0, "tracing-thresh", &ftrace.tracing_thresh,
 		     "Only show functions of which the duration is greater than <n>µs"),
+	OPT_BOOLEAN(0, "trace-children", &ftrace.trace_children,
+		    "Trace children processes"),
 	OPT_END()
 	};
 
-- 
2.25.1

[PATCH 13/19] perf ftrace: add option '-b/--buffer-size' to set per-cpu buffer size

[Changbin Du]

This adds an option '-b/--buffer-size' to allow us set the size of per-cpu
tracing buffer.

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 45 ++++++++++++++++++++++++++++---------
 1 file changed, 35 insertions(+), 10 deletions(-)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index 8fd95c109fe8..a93fbdac6aa4 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -47,6 +47,7 @@ struct perf_ftrace {
 	bool			long_info;
 	unsigned		tracing_thresh;
 	bool			trace_children;
+	unsigned		buffer_size_kb;
 };
 
 struct filter_entry {
@@ -187,6 +188,17 @@ static int read_tracing_file_to_stdout(const char *name)
 	return ret;
 }
 
+static int write_tracing_file_int(const char *name, int value)
+{
+	char buf[16];
+
+	snprintf(buf, sizeof(buf), "%d", value);
+	if (write_tracing_file(name, buf) < 0)
+		return -1;
+
+	return 0;
+}
+
 static int reset_tracing_cpu(void);
 static void reset_tracing_filters(void);
 
@@ -360,8 +372,6 @@ static void reset_tracing_filters(void)
 
 static int set_tracing_depth(struct perf_ftrace *ftrace)
 {
-	char buf[16];
-
 	if (ftrace->graph_depth == 0)
 		return 0;
 
@@ -370,9 +380,7 @@ static int set_tracing_depth(struct perf_ftrace *ftrace)
 		return -1;
 	}
 
-	snprintf(buf, sizeof(buf), "%d", ftrace->graph_depth);
-
-	if (write_tracing_file("max_graph_depth", buf) < 0)
+	if (write_tracing_file_int("max_graph_depth", ftrace->graph_depth) < 0)
 		return -1;
 
 	return 0;
@@ -419,14 +427,10 @@ static int set_tracing_long_info(struct perf_ftrace *ftrace)
 
 static int set_tracing_thresh(struct perf_ftrace *ftrace)
 {
-	char buf[16];
-
 	if (ftrace->tracing_thresh == 0)
 		return 0;
 
-	snprintf(buf, sizeof(buf), "%d", ftrace->tracing_thresh);
-
-	if (write_tracing_file("tracing_thresh", buf) < 0)
+	if (write_tracing_file_int("tracing_thresh", ftrace->tracing_thresh) < 0)
 		return -1;
 
 	return 0;
@@ -454,6 +458,20 @@ static int set_tracing_trace_children(struct perf_ftrace *ftrace)
 	return 0;
 }
 
+static int set_tracing_buffer_size_kb(struct perf_ftrace *ftrace)
+{
+	int ret;
+
+	if (ftrace->buffer_size_kb == 0)
+		return 0;
+
+	ret = write_tracing_file_int("buffer_size_kb", ftrace->buffer_size_kb);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
 static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 {
 	char *trace_file;
@@ -557,6 +575,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 		goto out_reset;
 	}
 
+	if (set_tracing_buffer_size_kb(ftrace) < 0) {
+		pr_err("failed to set tracing per-cpu buffer size\n");
+		goto out_reset;
+	}
+
 	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
 		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
 		goto out_reset;
@@ -706,6 +729,8 @@ int cmd_ftrace(int argc, const char **argv)
 		     "Only show functions of which the duration is greater than <n>µs"),
 	OPT_BOOLEAN(0, "trace-children", &ftrace.trace_children,
 		    "Trace children processes"),
+	OPT_UINTEGER('b', "buffer-size", &ftrace.buffer_size_kb,
+		     "size of per cpu buffer in kb"),
 	OPT_END()
 	};
 
-- 
2.25.1

[PATCH 14/19] perf ftrace: add option -P/--no-pager to disable pager

[Changbin Du]

Sometimes we want perf displays trace immediately. So this adds an option
'-P/--no-pager' to disable pager if needed.

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index a93fbdac6aa4..64c22f367ba2 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -48,6 +48,7 @@ struct perf_ftrace {
 	unsigned		tracing_thresh;
 	bool			trace_children;
 	unsigned		buffer_size_kb;
+	bool			no_pager;
 };
 
 struct filter_entry {
@@ -56,6 +57,7 @@ struct filter_entry {
 };
 
 static volatile int workload_exec_errno;
+static bool interrupted;
 static bool done;
 
 static void sig_handler(int sig __maybe_unused)
@@ -63,6 +65,12 @@ static void sig_handler(int sig __maybe_unused)
 	done = true;
 }
 
+static void sig_handler_int(int sig __maybe_unused)
+{
+	sig_handler(sig);
+	interrupted = 1;
+}
+
 /*
  * perf_evlist__prepare_workload will send a SIGUSR1 if the fork fails, since
  * we asked by setting its exec_error to the function below,
@@ -492,7 +500,7 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 		return -1;
 	}
 
-	signal(SIGINT, sig_handler);
+	signal(SIGINT, sig_handler_int);
 	signal(SIGUSR1, sig_handler);
 	signal(SIGCHLD, sig_handler);
 	signal(SIGPIPE, sig_handler);
@@ -585,7 +593,8 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 		goto out_reset;
 	}
 
-	setup_pager();
+	if (!ftrace->no_pager)
+		setup_pager();
 
 	trace_file = get_tracing_file("trace_pipe");
 	if (!trace_file) {
@@ -636,7 +645,7 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 	}
 
 	/* read remaining buffer contents */
-	while (true) {
+	while (true && !interrupted) {
 		int n = read(trace_fd, buf, sizeof(buf));
 		if (n <= 0)
 			break;
@@ -731,6 +740,8 @@ int cmd_ftrace(int argc, const char **argv)
 		    "Trace children processes"),
 	OPT_UINTEGER('b', "buffer-size", &ftrace.buffer_size_kb,
 		     "size of per cpu buffer in kb"),
+	OPT_BOOLEAN('P', "no-pager", &ftrace.no_pager,
+		    "Do not use pager"),
 	OPT_END()
 	};
 
-- 
2.25.1

[PATCH 15/19] perf ftrace: show trace column header

[Changbin Du]

This makes perf-ftrace display column header before printing trace.

$ sudo perf ftrace
\# tracer: function
\#
\# entries-in-buffer/entries-written: 0/0   #P:8
\#
\#           TASK-PID     CPU#   TIMESTAMP  FUNCTION
\#              | |         |       |         |
           <...>-9246  [006]  10726.262760: mutex_unlock <-rb_simple_write
           <...>-9246  [006]  10726.262764: __fsnotify_parent <-vfs_write
           <...>-9246  [006]  10726.262765: fsnotify <-vfs_write
           <...>-9246  [006]  10726.262766: __sb_end_write <-vfs_write
           <...>-9246  [006]  10726.262767: fpregs_assert_state_consistent <-do_syscall_64

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index 64c22f367ba2..0b39b6a88026 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -614,6 +614,9 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 	fcntl(trace_fd, F_SETFL, O_NONBLOCK);
 	pollfd.fd = trace_fd;
 
+	/* display column headers */
+	read_tracing_file_to_stdout("trace");
+
 	if (write_tracing_file("tracing_on", "1") < 0) {
 		pr_err("can't enable tracing\n");
 		goto out_close_fd;
-- 
2.25.1

[PATCH 16/19] perf ftrace: add option -t/--tid to filter by thread id

[Changbin Du]

This allows us to trace single thread instead of the whole process.

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index 0b39b6a88026..8d04e5afe2d3 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -707,6 +707,8 @@ int cmd_ftrace(int argc, const char **argv)
 		    "List available functions to filter"),
 	OPT_STRING('p', "pid", &ftrace.target.pid, "pid",
 		   "trace on existing process id"),
+	OPT_STRING('t', "tid", &ftrace.target.tid, "tid",
+		   "trace on existing thread id (exclusive to --pid)"),
 	OPT_INCR('v', "verbose", &verbose,
 		 "be more verbose"),
 	OPT_BOOLEAN('a', "all-cpus", &ftrace.target.system_wide,
-- 
2.25.1

[PATCH 17/19] perf ftrace: add option -d/--delay to delay tracing

[Changbin Du]

This adds an option '-d/--delay' to allow us to start tracing some
times later after workload is launched.

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index 8d04e5afe2d3..d376b37c53fc 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -49,6 +49,7 @@ struct perf_ftrace {
 	bool			trace_children;
 	unsigned		buffer_size_kb;
 	bool			no_pager;
+	unsigned		initial_delay;
 };
 
 struct filter_entry {
@@ -617,13 +618,23 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 	/* display column headers */
 	read_tracing_file_to_stdout("trace");
 
-	if (write_tracing_file("tracing_on", "1") < 0) {
-		pr_err("can't enable tracing\n");
-		goto out_close_fd;
+	if (!ftrace->initial_delay) {
+		if (write_tracing_file("tracing_on", "1") < 0) {
+			pr_err("can't enable tracing\n");
+			goto out_close_fd;
+		}
 	}
 
 	perf_evlist__start_workload(ftrace->evlist);
 
+	if (ftrace->initial_delay) {
+		usleep(ftrace->initial_delay * 1000);
+		if (write_tracing_file("tracing_on", "1") < 0) {
+			pr_err("can't enable tracing\n");
+			goto out_close_fd;
+		}
+	}
+
 	while (!done) {
 		if (poll(&pollfd, 1, -1) < 0)
 			break;
@@ -747,6 +758,8 @@ int cmd_ftrace(int argc, const char **argv)
 		     "size of per cpu buffer in kb"),
 	OPT_BOOLEAN('P', "no-pager", &ftrace.no_pager,
 		    "Do not use pager"),
+	OPT_UINTEGER('d', "delay", &ftrace.initial_delay,
+		     "Wait <n> ms before tracing"),
 	OPT_END()
 	};
 
-- 
2.25.1

[PATCH 18/19] perf ftrace: add option --latency-format to display more info about delay

[Changbin Du]

This is for the function graph tracer to display more info about latency.
The execution context is shown in this case.

$ sudo perf ftrace -G --latency-format
\# tracer: function_graph
\#
 1)  .... |   0.992 us    |  mutex_unlock();
 1)  .... |   1.404 us    |  __fsnotify_parent();
 1)  .... |   1.023 us    |  fsnotify();
 1)  .... |   0.335 us    |  __sb_end_write();
 1)  d... |   0.439 us    |  fpregs_assert_state_consistent();
 1)  d... |               |  do_syscall_64() {
 1)  .... |               |    __x64_sys_close() {

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index d376b37c53fc..fd8e2f305136 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -44,6 +44,7 @@ struct perf_ftrace {
 	bool			nosleep_time;
 	bool			nofuncgraph_irqs;
 	bool			funcgraph_tail;
+	bool			latency_format;
 	bool			long_info;
 	unsigned		tracing_thresh;
 	bool			trace_children;
@@ -217,6 +218,7 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
 	write_tracing_option_file("sleep-time", "1");
 	write_tracing_option_file("funcgraph-irqs", "1");
 	write_tracing_option_file("funcgraph-tail", "0");
+	write_tracing_option_file("latency-format", "0");
 	write_tracing_option_file("funcgraph-proc", "0");
 	write_tracing_option_file("funcgraph-abstime", "0");
 	write_tracing_option_file("irq-info", "0");
@@ -456,6 +458,17 @@ static int set_tracing_funcgraph_tail(struct perf_ftrace *ftrace)
 	return 0;
 }
 
+static int set_tracing_latency_format(struct perf_ftrace *ftrace)
+{
+	if (!ftrace->latency_format)
+		return 0;
+
+	if (write_tracing_option_file("latency-format", "1") < 0)
+		return -1;
+
+	return 0;
+}
+
 static int set_tracing_trace_children(struct perf_ftrace *ftrace)
 {
 	if (!ftrace->trace_children)
@@ -579,6 +592,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 		goto out_reset;
 	}
 
+	if (set_tracing_latency_format(ftrace) < 0) {
+		pr_err("failed to set tracing option latency-format\n");
+		goto out_reset;
+	}
+
 	if (set_tracing_trace_children(ftrace) < 0) {
 		pr_err("failed to set tracing option function-fork\n");
 		goto out_reset;
@@ -748,6 +766,8 @@ int cmd_ftrace(int argc, const char **argv)
 		    "Ignore functions that happen inside interrupt (function_graph only)"),
 	OPT_BOOLEAN(0, "funcgraph-tail", &ftrace.funcgraph_tail,
 		    "Show function tails comment (function_graph only)"),
+	OPT_BOOLEAN(0, "latency-format", &ftrace.latency_format,
+		    "displays additional information about the latency (function_graph only)"),
 	OPT_BOOLEAN('l', "long-info", &ftrace.long_info,
 		    "Show process names, PIDs, timestamps, irq-info if available"),
 	OPT_UINTEGER(0, "tracing-thresh", &ftrace.tracing_thresh,
-- 
2.25.1

[PATCH 19/19] perf ftrace: add change log

[Changbin Du]

Add a change log after previous enhancements.

Signed-off-by: Changbin Du <changbin.du@gmail.com>
---
 tools/perf/builtin-ftrace.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index fd8e2f305136..8170746f94fd 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -3,6 +3,7 @@
  * builtin-ftrace.c
  *
  * Copyright (c) 2013  LG Electronics,  Namhyung Kim <namhyung@kernel.org>
+ * Copyright (c) 2020  Changbin Du <changbin.du@gmail.com>, significant enhancement.
  */
 
 #include "builtin.h"
-- 
2.25.1

Re: [PATCH 00/19] perf: ftrace enhancement

[Steven Rostedt]

On Sun, 10 May 2020 23:06:09 +0800
Changbin Du <changbin.du@gmail.com> wrote:

> The perf has basic kernel ftrace support but lack support of most tracing
> options. This serias is target to enhance the perf ftrace functionality so
> that we can make full use of kernel ftrace with only perf.
> 
> In general, this serias be cataloged into two main changes:
>   1) Improve usability of existing functions. For example, we don't need to type
>      extra option to select the tracer.
>   2) Add new options to support all other ftrace functions.
> 
> Here is a glance of all ftrace functions with this serias:
>   * - improved existing options.
>   + - new added options.
> 
> $ sudo perf ftrace -h
> 
>  Usage: perf ftrace [<options>] [<command>]
>     or: perf ftrace [<options>] -- <command> [<options>]
> 
> *   -a, --all-cpus        system-wide collection from all CPUs
> +   -b, --buffer-size <n>
>                           size of per cpu buffer in kb
>     -C, --cpu <cpu>       list of cpus to monitor
> +   -d, --delay <n>       Wait <n> ms before tracing
>     -D, --graph-depth <n>
>                           Max depth for function graph tracer
> *   -G, --graph-funcs <func>
>                           Set graph filter on given functions (imply to use function_graph tracer)
>     -g, --nograph-funcs <func>
>                           Set nograph filter on given functions (imply to use function_graph tracer)
> +   -L, --list-functions  List available functions to filter
> +   -l, --long-info       Show process names, PIDs, timestamps, irq-info if available
>     -N, --notrace-funcs <func>
>                           do not trace given functions
> +   -P, --no-pager        Do not use pager
>     -p, --pid <pid>       trace on existing process id
> +   -s, --func-stack-trace
>                           Show kernel stack trace for function tracer
> +   -t, --tid <tid>       trace on existing thread id (exclusive to --pid)
>     -T, --trace-funcs <func>
>                           trace given functions only
> +   -u, --userstacktrace  Show stacktrace of the current user space thread
>     -v, --verbose         be more verbose
> +       --funcgraph-tail  Show function tails comment (function_graph only)
> +       --latency-format  displays additional information about the latency (function_graph only)
> +       --nofuncgraph-irqs
>                           Ignore functions that happen inside interrupt (function_graph only)
> +       --nosleep-time    Measure on-CPU time only (function_graph only)
> +       --trace-children  Trace children processes
> +       --tracing-thresh <n>
>                           Only show functions of which the duration is greater than <n>µs
> 

Note, we are working on making more of the trace-cmd functionality into
libraries. See this work here:

https://lore.kernel.org/r/20191219113502.28964-2-tz.stoyanov@gmail.com

Which introduces a libtracefs, that is to handle all the work needed to
interact with the tracefs directory. This will also be useful for perf
to read the event directory without having to open code that work.

I'm all for giving perf the functionality of ftrace, but I would like
to have it do so with a more generic solution that other tools could
benefit from as well.

Thanks!

-- Steve

Re: [PATCH 00/19] perf: ftrace enhancement

[Arnaldo Melo]

On May 10, 2020 12:23:36 PM GMT-03:00, Steven Rostedt <rostedt@goodmis.org> wrote:
>On Sun, 10 May 2020 23:06:09 +0800
>Changbin Du <changbin.du@gmail.com> wrote:
>
>> The perf has basic kernel ftrace support but lack support of most
>tracing
>> options. This serias is target to enhance the perf ftrace
>functionality so
>> that we can make full use of kernel ftrace with only perf.
>> 
>> In general, this serias be cataloged into two main changes:
>>   1) Improve usability of existing functions. For example, we don't
>need to type
>>      extra option to select the tracer.
>>   2) Add new options to support all other ftrace functions.
>> 
>> Here is a glance of all ftrace functions with this serias:
>>   * - improved existing options.
>>   + - new added options.
>> 
>> $ sudo perf ftrace -h
>> 
>>  Usage: perf ftrace [<options>] [<command>]
>>     or: perf ftrace [<options>] -- <command> [<options>]
>> 
>> *   -a, --all-cpus        system-wide collection from all CPUs
>> +   -b, --buffer-size <n>
>>                           size of per cpu buffer in kb
>>     -C, --cpu <cpu>       list of cpus to monitor
>> +   -d, --delay <n>       Wait <n> ms before tracing
>>     -D, --graph-depth <n>
>>                           Max depth for function graph tracer
>> *   -G, --graph-funcs <func>
>>                           Set graph filter on given functions (imply
>to use function_graph tracer)
>>     -g, --nograph-funcs <func>
>>                           Set nograph filter on given functions
>(imply to use function_graph tracer)
>> +   -L, --list-functions  List available functions to filter
>> +   -l, --long-info       Show process names, PIDs, timestamps,
>irq-info if available
>>     -N, --notrace-funcs <func>
>>                           do not trace given functions
>> +   -P, --no-pager        Do not use pager
>>     -p, --pid <pid>       trace on existing process id
>> +   -s, --func-stack-trace
>>                           Show kernel stack trace for function tracer
>> +   -t, --tid <tid>       trace on existing thread id (exclusive to
>--pid)
>>     -T, --trace-funcs <func>
>>                           trace given functions only
>> +   -u, --userstacktrace  Show stacktrace of the current user space
>thread
>>     -v, --verbose         be more verbose
>> +       --funcgraph-tail  Show function tails comment (function_graph
>only)
>> +       --latency-format  displays additional information about the
>latency (function_graph only)
>> +       --nofuncgraph-irqs
>>                           Ignore functions that happen inside
>interrupt (function_graph only)
>> +       --nosleep-time    Measure on-CPU time only (function_graph
>only)
>> +       --trace-children  Trace children processes
>> +       --tracing-thresh <n>
>>                           Only show functions of which the duration
>is greater than <n>µs
>> 
>
>Note, we are working on making more of the trace-cmd functionality into
>libraries. See this work here:
>
>https://lore.kernel.org/r/20191219113502.28964-2-tz.stoyanov@gmail.com
>
>Which introduces a libtracefs, that is to handle all the work needed to
>interact with the tracefs directory. This will also be useful for perf
>to read the event directory without having to open code that work.
>
>I'm all for giving perf the functionality of ftrace, but I would like
>to have it do so with a more generic solution that other tools could
>benefit from as well.
>

I think in time things will fall into place, 'perf ftrace' has potential as a way for perf users to have a familiar interface to ftrace, and that I think it's its raison d'être, in time well go on finding ways to have common code.

Tomorrow I'll go over this patch set,

Thanks,

- Arnaldo

>Thanks!
>
>-- Steve

-- 
Sent from my Android device with K-9 Mail. Please excuse my brevity.

Re: [PATCH 00/19] perf: ftrace enhancement

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:09PM +0800, Changbin Du escreveu:
> The perf has basic kernel ftrace support but lack support of most tracing
> options. This serias is target to enhance the perf ftrace functionality so
> that we can make full use of kernel ftrace with only perf.

Quite a lot! Going thru it now.

- Arnaldo
 
> In general, this serias be cataloged into two main changes:
>   1) Improve usability of existing functions. For example, we don't need to type
>      extra option to select the tracer.
>   2) Add new options to support all other ftrace functions.
> 
> Here is a glance of all ftrace functions with this serias:
>   * - improved existing options.
>   + - new added options.
> 
> $ sudo perf ftrace -h
> 
>  Usage: perf ftrace [<options>] [<command>]
>     or: perf ftrace [<options>] -- <command> [<options>]
> 
> *   -a, --all-cpus        system-wide collection from all CPUs
> +   -b, --buffer-size <n>
>                           size of per cpu buffer in kb
>     -C, --cpu <cpu>       list of cpus to monitor
> +   -d, --delay <n>       Wait <n> ms before tracing
>     -D, --graph-depth <n>
>                           Max depth for function graph tracer
> *   -G, --graph-funcs <func>
>                           Set graph filter on given functions (imply to use function_graph tracer)
>     -g, --nograph-funcs <func>
>                           Set nograph filter on given functions (imply to use function_graph tracer)
> +   -L, --list-functions  List available functions to filter
> +   -l, --long-info       Show process names, PIDs, timestamps, irq-info if available
>     -N, --notrace-funcs <func>
>                           do not trace given functions
> +   -P, --no-pager        Do not use pager
>     -p, --pid <pid>       trace on existing process id
> +   -s, --func-stack-trace
>                           Show kernel stack trace for function tracer
> +   -t, --tid <tid>       trace on existing thread id (exclusive to --pid)
>     -T, --trace-funcs <func>
>                           trace given functions only
> +   -u, --userstacktrace  Show stacktrace of the current user space thread
>     -v, --verbose         be more verbose
> +       --funcgraph-tail  Show function tails comment (function_graph only)
> +       --latency-format  displays additional information about the latency (function_graph only)
> +       --nofuncgraph-irqs
>                           Ignore functions that happen inside interrupt (function_graph only)
> +       --nosleep-time    Measure on-CPU time only (function_graph only)
> +       --trace-children  Trace children processes
> +       --tracing-thresh <n>
>                           Only show functions of which the duration is greater than <n>µs
> 
> 
> Changbin Du (19):
>   perf ftrace: trace system wide if no target is given
>   perf ftrace: detect workload failure
>   perf ftrace: select function/function_graph tracer automatically
>   perf ftrace: add support for tracing option 'func_stack_trace'
>   perf ftrace: add option '-l/--list-functions' to list available
>     functions
>   perf ftrace: add support for trace option sleep-time
>   perf ftrace: add support for trace option funcgraph-irqs
>   perf ftrace: add option -l/--long-info to show more info
>   perf ftrace: add support for trace option tracing_thresh
>   perf ftrace: add support for trace option funcgraph-tail
>   perf ftrace: add option '-u/--userstacktrace' to show userspace
>     stacktrace
>   perf ftrace: add support for tracing children processes
>   perf ftrace: add option '-b/--buffer-size' to set per-cpu buffer size
>   perf ftrace: add option -P/--no-pager to disable pager
>   perf ftrace: show trace column header
>   perf ftrace: add option -t/--tid to filter by thread id
>   perf ftrace: add option -d/--delay to delay tracing
>   perf ftrace: add option --latency-format to display more info about
>     delay
>   perf ftrace: add change log
> 
>  tools/perf/Documentation/perf-config.txt |   5 -
>  tools/perf/builtin-ftrace.c              | 388 ++++++++++++++++++++---
>  2 files changed, 345 insertions(+), 48 deletions(-)
> 
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 01/19] perf ftrace: trace system wide if no target is given

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:10PM +0800, Changbin Du escreveu:
> This align ftrace to other perf sub-commands that if no target specified
> then we trace all functions.


Thanks, applied,

- Arnaldo
 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index d5adc417a4ca..11fc02037899 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -493,7 +493,7 @@ int cmd_ftrace(int argc, const char **argv)
>  	argc = parse_options(argc, argv, ftrace_options, ftrace_usage,
>  			    PARSE_OPT_STOP_AT_NON_OPTION);
>  	if (!argc && target__none(&ftrace.target))
> -		usage_with_options(ftrace_usage, ftrace_options);
> +		ftrace.target.system_wide = true;
>  
>  	ret = target__validate(&ftrace.target);
>  	if (ret) {
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 02/19] perf ftrace: detect workload failure

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:11PM +0800, Changbin Du escreveu:
> Currently there's no error message prompted if we failed to start workload.
> And we still get some trace which is confusing. Let's tell users what
> happened.

Applied, but please in the future add Before and After output of the
commands for these to be clear even to my 4yo kid:

commit 2a5193e460eaf8792a87f678e3c5f78c88db2123
Author: Changbin Du <changbin.du@intel.com>
Date:   Sun May 10 23:06:11 2020 +0800

    perf ftrace: Detect workload failure
    
    Currently there's no error message prompted if we failed to start
    workload.  And we still get some trace which is confusing. Let's tell
    users what happened.
    
    Committer testing:
    
    Before:
    
        # perf ftrace nonsense |& head
         5)               |  switch_mm_irqs_off() {
         5)   0.400 us    |    load_new_mm_cr3();
         5)   3.261 us    |  }
         ------------------------------------------
         5)    <idle>-0    =>   <...>-3494
         ------------------------------------------
    
         5)               |  finish_task_switch() {
         5)   ==========> |
         5)               |    smp_irq_work_interrupt() {
        # type nonsense
        -bash: type: nonsense: not found
        #
    
    After:
    
      # perf ftrace nonsense |& head
      workload failed: No such file or directory
      # type nonsense
      -bash: type: nonsense: not found
      #
    
    Signed-off-by: Changbin Du <changbin.du@gmail.com>
    Tested-by: Arnaldo Carvalho de Melo <acme@redhat.com>
    Cc: Jiri Olsa <jolsa@redhat.com>
    Cc: Namhyung Kim <namhyung@kernel.org>
    Cc: Peter Zijlstra <peterz@infradead.org>
    Cc: Steven Rostedt (VMware) <rostedt@goodmis.org>
    Link: http://lore.kernel.org/lkml/20200510150628.16610-3-changbin.du@gmail.com
    Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>

diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
index b7d3fb5fa8b1..2bfc1b0db536 100644
--- a/tools/perf/builtin-ftrace.c
+++ b/tools/perf/builtin-ftrace.c
@@ -45,6 +45,7 @@ struct filter_entry {
 	char			name[];
 };
 
+static volatile int workload_exec_errno;
 static bool done;
 
 static void sig_handler(int sig __maybe_unused)
@@ -63,7 +64,7 @@ static void ftrace__workload_exec_failed_signal(int signo __maybe_unused,
 						siginfo_t *info __maybe_unused,
 						void *ucontext __maybe_unused)
 {
-	/* workload_exec_errno = info->si_value.sival_int; */
+	workload_exec_errno = info->si_value.sival_int;
 	done = true;
 }
 
@@ -383,6 +384,14 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 
 	write_tracing_file("tracing_on", "0");
 
+	if (workload_exec_errno) {
+		const char *emsg = str_error_r(workload_exec_errno, buf, sizeof(buf));
+		/* flush stdout first so below error msg appears at the end. */
+		fflush(stdout);
+		pr_err("workload failed: %s\n", emsg);
+		goto out_close_fd;
+	}
+
 	/* read remaining buffer contents */
 	while (true) {
 		int n = read(trace_fd, buf, sizeof(buf));
@@ -397,7 +406,7 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
 out_reset:
 	reset_tracing_files(ftrace);
 out:
-	return done ? 0 : -1;
+	return (done && !workload_exec_errno) ? 0 : -1;
 }
 
 static int perf_ftrace_config(const char *var, const char *value, void *cb)

Re: [PATCH 03/19] perf ftrace: select function/function_graph tracer automatically

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:12PM +0800, Changbin Du escreveu:
> The '-g/-G' options have already implied function_graph tracer should be
> used instead of function tracer. So the extra option '--tracer' can be
> killed.

Well, since it is there, lets not remove it, you can even add a
"Deprecated" message to it and then after some time, remove it.

Also, in this area, -g is used for asking for callchains in other perf
tools, so its a candidate for a change in behaviour, as well with some
deprecation warning for a while, using --call-graph for a while, etc.

I.e. we want it to have a similar set of options as the other perf
tools, so that switching from, say:

  # perf trace <options> workload

To:

  # perf ftrace <options> workload

Would provide a similar experience, only that the first is limited to
syscalls, the other to kernel functions.

- Arnaldo

 
> This patch changes the behavior as below:
>   - By default, function tracer is used.
>   - If '-g' or '-G' option is on, then function_graph tracer is used.
>   - The perf configuration item 'ftrace.tracer' is removed.
>   - The default filter for -G is to trace all functions.
> 
> Here are some examples.
> 
> This will start tracing all functions using function tracer:
>   $ sudo perf ftrace
> 
> This will trace all functions using function graph tracer:
>   $ sudo perf ftrace -G
> 
> This will trace function vfs_read using function graph tracer:
>   $ sudo perf ftrace -G vfs_read
> 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/Documentation/perf-config.txt |  5 ---
>  tools/perf/builtin-ftrace.c              | 39 ++++++------------------
>  2 files changed, 9 insertions(+), 35 deletions(-)
> 
> diff --git a/tools/perf/Documentation/perf-config.txt b/tools/perf/Documentation/perf-config.txt
> index f16d8a71d3f5..fad6c48ed76a 100644
> --- a/tools/perf/Documentation/perf-config.txt
> +++ b/tools/perf/Documentation/perf-config.txt
> @@ -612,11 +612,6 @@ trace.*::
>  		"libbeauty", the default, to use the same argument beautifiers used in the
>  		strace-like sys_enter+sys_exit lines.
>  
> -ftrace.*::
> -	ftrace.tracer::
> -		Can be used to select the default tracer. Possible values are
> -		'function' and 'function_graph'.
> -
>  llvm.*::
>  	llvm.clang-path::
>  		Path to clang. If omit, search it from $PATH.
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index 5584f8dec25d..57e656c35d28 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -27,7 +27,7 @@
>  #include "util/cap.h"
>  #include "util/config.h"
>  
> -#define DEFAULT_TRACER  "function_graph"
> +#define DEFAULT_TRACER  "function"
>  
>  struct perf_ftrace {
>  	struct evlist		*evlist;
> @@ -408,26 +408,6 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  	return (done && !workload_exec_errno) ? 0 : -1;
>  }
>  
> -static int perf_ftrace_config(const char *var, const char *value, void *cb)
> -{
> -	struct perf_ftrace *ftrace = cb;
> -
> -	if (!strstarts(var, "ftrace."))
> -		return 0;
> -
> -	if (strcmp(var, "ftrace.tracer"))
> -		return -1;
> -
> -	if (!strcmp(value, "function_graph") ||
> -	    !strcmp(value, "function")) {
> -		ftrace->tracer = value;
> -		return 0;
> -	}
> -
> -	pr_err("Please select \"function_graph\" (default) or \"function\"\n");
> -	return -1;
> -}
> -
>  static int parse_filter_func(const struct option *opt, const char *str,
>  			     int unset __maybe_unused)
>  {
> @@ -467,8 +447,6 @@ int cmd_ftrace(int argc, const char **argv)
>  		NULL
>  	};
>  	const struct option ftrace_options[] = {
> -	OPT_STRING('t', "tracer", &ftrace.tracer, "tracer",
> -		   "tracer to use: function_graph(default) or function"),
>  	OPT_STRING('p', "pid", &ftrace.target.pid, "pid",
>  		   "trace on existing process id"),
>  	OPT_INCR('v', "verbose", &verbose,
> @@ -481,10 +459,12 @@ int cmd_ftrace(int argc, const char **argv)
>  		     "trace given functions only", parse_filter_func),
>  	OPT_CALLBACK('N', "notrace-funcs", &ftrace.notrace, "func",
>  		     "do not trace given functions", parse_filter_func),
> -	OPT_CALLBACK('G', "graph-funcs", &ftrace.graph_funcs, "func",
> -		     "Set graph filter on given functions", parse_filter_func),
> +	OPT_CALLBACK_DEFAULT('G', "graph-funcs", &ftrace.graph_funcs, "func",
> +		     "Set graph filter on given functions (imply to use function_graph tracer)",
> +		     parse_filter_func, "*"),
>  	OPT_CALLBACK('g', "nograph-funcs", &ftrace.nograph_funcs, "func",
> -		     "Set nograph filter on given functions", parse_filter_func),
> +		     "Set nograph filter on given functions (imply to use function_graph tracer)",
> +		     parse_filter_func),
>  	OPT_INTEGER('D', "graph-depth", &ftrace.graph_depth,
>  		    "Max depth for function graph tracer"),
>  	OPT_END()
> @@ -495,15 +475,14 @@ int cmd_ftrace(int argc, const char **argv)
>  	INIT_LIST_HEAD(&ftrace.graph_funcs);
>  	INIT_LIST_HEAD(&ftrace.nograph_funcs);
>  
> -	ret = perf_config(perf_ftrace_config, &ftrace);
> -	if (ret < 0)
> -		return -1;
> -
>  	argc = parse_options(argc, argv, ftrace_options, ftrace_usage,
>  			    PARSE_OPT_STOP_AT_NON_OPTION);
>  	if (!argc && target__none(&ftrace.target))
>  		ftrace.target.system_wide = true;
>  
> +	if (!list_empty(&ftrace.graph_funcs) || !list_empty(&ftrace.nograph_funcs))
> +		ftrace.tracer = "function_graph";
> +
>  	ret = target__validate(&ftrace.target);
>  	if (ret) {
>  		char errbuf[512];
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 04/19] perf ftrace: add support for tracing option 'func_stack_trace'

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:13PM +0800, Changbin Du escreveu:
> This adds support to display call trace for function tracer. To do this,
> just specify a '-s' option.
> 
> $ sudo perf ftrace -T vfs_read -s
>  iio-sensor-prox-855   [003]   6168.369657: vfs_read <-ksys_read
>  iio-sensor-prox-855   [003]   6168.369677: <stack trace>
>  => vfs_read
>  => ksys_read
>  => __x64_sys_read
>  => do_syscall_64
>  => entry_SYSCALL_64_after_hwframe
>  ...

So, for the reason stated in my response to the patch 03/19, I think we
should start here with --call-graph for this, instead of
--func-stack-trace. And for now don't provide a one-letter equivalent.

- Arnaldo
 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 38 +++++++++++++++++++++++++++++++++++++
>  1 file changed, 38 insertions(+)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index 57e656c35d28..1d30c2d5f88b 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -38,6 +38,7 @@ struct perf_ftrace {
>  	struct list_head	graph_funcs;
>  	struct list_head	nograph_funcs;
>  	int			graph_depth;
> +	bool			func_stack_trace;
>  };
>  
>  struct filter_entry {
> @@ -128,9 +129,27 @@ static int append_tracing_file(const char *name, const char *val)
>  	return __write_tracing_file(name, val, true);
>  }
>  
> +static int write_tracing_option_file(const char *name, const char *val)
> +{
> +	char *file;
> +	int ret;
> +
> +	if (asprintf(&file, "options/%s", name) < 0)
> +		return -1;
> +
> +	ret = __write_tracing_file(file, val, false);
> +	free(file);
> +	return ret;
> +}
> +
>  static int reset_tracing_cpu(void);
>  static void reset_tracing_filters(void);
>  
> +static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
> +{
> +	write_tracing_option_file("func_stack_trace", "0");
> +}
> +
>  static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
>  {
>  	if (write_tracing_file("tracing_on", "0") < 0)
> @@ -149,6 +168,7 @@ static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
>  		return -1;
>  
>  	reset_tracing_filters();
> +	reset_tracing_options(ftrace);
>  	return 0;
>  }
>  
> @@ -204,6 +224,17 @@ static int set_tracing_cpu(struct perf_ftrace *ftrace)
>  	return set_tracing_cpumask(cpumap);
>  }
>  
> +static int set_tracing_func_stack_trace(struct perf_ftrace *ftrace)
> +{
> +	if (!ftrace->func_stack_trace)
> +		return 0;
> +
> +	if (write_tracing_option_file("func_stack_trace", "1") < 0)
> +		return -1;
> +
> +	return 0;
> +}
> +
>  static int reset_tracing_cpu(void)
>  {
>  	struct perf_cpu_map *cpumap = perf_cpu_map__new(NULL);
> @@ -326,6 +357,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  		goto out_reset;
>  	}
>  
> +	if (set_tracing_func_stack_trace(ftrace) < 0) {
> +		pr_err("failed to set tracing option func_stack_trace\n");
> +		goto out_reset;
> +	}
> +
>  	if (set_tracing_filters(ftrace) < 0) {
>  		pr_err("failed to set tracing filters\n");
>  		goto out_reset;
> @@ -459,6 +495,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		     "trace given functions only", parse_filter_func),
>  	OPT_CALLBACK('N', "notrace-funcs", &ftrace.notrace, "func",
>  		     "do not trace given functions", parse_filter_func),
> +	OPT_BOOLEAN('s', "func-stack-trace", &ftrace.func_stack_trace,
> +		    "Show kernel stack trace for function tracer"),
>  	OPT_CALLBACK_DEFAULT('G', "graph-funcs", &ftrace.graph_funcs, "func",
>  		     "Set graph filter on given functions (imply to use function_graph tracer)",
>  		     parse_filter_func, "*"),
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 05/19] perf ftrace: add option '-l/--list-functions' to list available functions

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:14PM +0800, Changbin Du escreveu:
> This adds an option '-l/--list-functions' to list all available functions
> which is read from tracing file 'available_filter_functions'.

Here, in 'perf probe' we have:

[acme@five perf]$ perf probe -F tcp_* | head
tcp_abort
tcp_ack
tcp_ack_update_rtt.isra.0
tcp_add_backlog
tcp_add_reno_sack
tcp_adjust_pcount
tcp_alloc_md5sig_pool
tcp_any_retrans_done.part.0
tcp_assign_congestion_control
tcp_bpf_clone
[acme@five perf]$ perf probe -h -F

 Usage: perf probe [<options>] 'PROBEDEF' ['PROBEDEF' ...]
    or: perf probe [<options>] --add 'PROBEDEF' [--add 'PROBEDEF' ...]
    or: perf probe [<options>] --del '[GROUP:]EVENT' ...
    or: perf probe --list [GROUP:]EVENT ...
    or: perf probe [<options>] --line 'LINEDESC'
    or: perf probe [<options>] --vars 'PROBEPOINT'
    or: perf probe [<options>] --funcs

    -F, --funcs <[FILTER]>
                          Show potential probe-able functions.

[acme@five perf]$

So I think we should align with this.

- Arnaldo
 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 43 +++++++++++++++++++++++++++++++++++++
>  1 file changed, 43 insertions(+)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index 1d30c2d5f88b..8133d910d5d8 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -33,6 +33,7 @@ struct perf_ftrace {
>  	struct evlist		*evlist;
>  	struct target		target;
>  	const char		*tracer;
> +	bool			list_avail_functions;
>  	struct list_head	filters;
>  	struct list_head	notrace;
>  	struct list_head	graph_funcs;
> @@ -142,6 +143,43 @@ static int write_tracing_option_file(const char *name, const char *val)
>  	return ret;
>  }
>  
> +static int read_tracing_file_to_stdout(const char *name)
> +{
> +	char buf[4096];
> +	char *file;
> +	int fd;
> +	int ret = -1;
> +
> +	file = get_tracing_file(name);
> +	if (!file) {
> +		pr_debug("cannot get tracing file: %s\n", name);
> +		return -1;
> +	}
> +
> +	fd = open(file, O_RDONLY);
> +	if (fd < 0) {
> +		pr_debug("cannot open tracing file: %s: %s\n",
> +			 name, str_error_r(errno, buf, sizeof(buf)));
> +		goto out;
> +	}
> +
> +	/* read contents to stdout */
> +	while (true) {
> +		int n = read(fd, buf, sizeof(buf));
> +		if (n <= 0)
> +			goto out_close;
> +		if (fwrite(buf, n, 1, stdout) != 1)
> +			goto out_close;
> +	}
> +	ret = 0;
> +
> +out_close:
> +	close(fd);
> +out:
> +	put_tracing_file(file);
> +	return ret;
> +}
> +
>  static int reset_tracing_cpu(void);
>  static void reset_tracing_filters(void);
>  
> @@ -332,6 +370,9 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  	signal(SIGCHLD, sig_handler);
>  	signal(SIGPIPE, sig_handler);
>  
> +	if (ftrace->list_avail_functions)
> +		return read_tracing_file_to_stdout("available_filter_functions");
> +
>  	if (reset_tracing_files(ftrace) < 0) {
>  		pr_err("failed to reset ftrace\n");
>  		goto out;
> @@ -483,6 +524,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		NULL
>  	};
>  	const struct option ftrace_options[] = {
> +	OPT_BOOLEAN('L', "list-functions", &ftrace.list_avail_functions,
> +		    "List available functions to filter"),
>  	OPT_STRING('p', "pid", &ftrace.target.pid, "pid",
>  		   "trace on existing process id"),
>  	OPT_INCR('v', "verbose", &verbose,
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 06/19] perf ftrace: add support for trace option sleep-time

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:15PM +0800, Changbin Du escreveu:
> This adds an option '--nosleep-time' which allow us only to measure
> on-CPU time. This option is function_graph tracer only.

This seems, for now, very specific to the function_graph tracer, so
perhaps we should have a:

	--function_graph_opts nosleep-time,other,another,etc

?

In time options that have equivalent in other perf tools can be
promoted?

- Arnaldo
 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 20 ++++++++++++++++++++
>  1 file changed, 20 insertions(+)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index 8133d910d5d8..d3fcf3b0b792 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -40,6 +40,7 @@ struct perf_ftrace {
>  	struct list_head	nograph_funcs;
>  	int			graph_depth;
>  	bool			func_stack_trace;
> +	bool			nosleep_time;
>  };
>  
>  struct filter_entry {
> @@ -186,6 +187,7 @@ static void reset_tracing_filters(void);
>  static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
>  {
>  	write_tracing_option_file("func_stack_trace", "0");
> +	write_tracing_option_file("sleep-time", "1");
>  }
>  
>  static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
> @@ -345,6 +347,17 @@ static int set_tracing_depth(struct perf_ftrace *ftrace)
>  	return 0;
>  }
>  
> +static int set_tracing_sleep_time(struct perf_ftrace *ftrace)
> +{
> +	if (!ftrace->nosleep_time)
> +		return 0;
> +
> +	if (write_tracing_option_file("sleep-time", "0") < 0)
> +		return -1;
> +
> +	return 0;
> +}
> +
>  static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  {
>  	char *trace_file;
> @@ -413,6 +426,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  		goto out_reset;
>  	}
>  
> +	if (set_tracing_sleep_time(ftrace) < 0) {
> +		pr_err("failed to set tracing option sleep-time\n");
> +		goto out_reset;
> +	}
> +
>  	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
>  		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
>  		goto out_reset;
> @@ -548,6 +566,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		     parse_filter_func),
>  	OPT_INTEGER('D', "graph-depth", &ftrace.graph_depth,
>  		    "Max depth for function graph tracer"),
> +	OPT_BOOLEAN(0, "nosleep-time", &ftrace.nosleep_time,
> +		    "Measure on-CPU time only (function_graph only)"),
>  	OPT_END()
>  	};
>  
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 07/19] perf ftrace: add support for trace option funcgraph-irqs

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:16PM +0800, Changbin Du escreveu:
> This adds an option '--nofuncgraph-irqs' to filter out functions executed
> in irq context.

Ditto
 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 20 ++++++++++++++++++++
>  1 file changed, 20 insertions(+)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index d3fcf3b0b792..b16600a16efa 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -41,6 +41,7 @@ struct perf_ftrace {
>  	int			graph_depth;
>  	bool			func_stack_trace;
>  	bool			nosleep_time;
> +	bool			nofuncgraph_irqs;
>  };
>  
>  struct filter_entry {
> @@ -188,6 +189,7 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
>  {
>  	write_tracing_option_file("func_stack_trace", "0");
>  	write_tracing_option_file("sleep-time", "1");
> +	write_tracing_option_file("funcgraph-irqs", "1");
>  }
>  
>  static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
> @@ -358,6 +360,17 @@ static int set_tracing_sleep_time(struct perf_ftrace *ftrace)
>  	return 0;
>  }
>  
> +static int set_tracing_funcgraph_irqs(struct perf_ftrace *ftrace)
> +{
> +	if (!ftrace->nofuncgraph_irqs)
> +		return 0;
> +
> +	if (write_tracing_option_file("funcgraph-irqs", "0") < 0)
> +		return -1;
> +
> +	return 0;
> +}
> +
>  static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  {
>  	char *trace_file;
> @@ -431,6 +444,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  		goto out_reset;
>  	}
>  
> +	if (set_tracing_funcgraph_irqs(ftrace) < 0) {
> +		pr_err("failed to set tracing option funcgraph-irqs\n");
> +		goto out_reset;
> +	}
> +
>  	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
>  		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
>  		goto out_reset;
> @@ -568,6 +586,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		    "Max depth for function graph tracer"),
>  	OPT_BOOLEAN(0, "nosleep-time", &ftrace.nosleep_time,
>  		    "Measure on-CPU time only (function_graph only)"),
> +	OPT_BOOLEAN(0, "nofuncgraph-irqs", &ftrace.nofuncgraph_irqs,
> +		    "Ignore functions that happen inside interrupt (function_graph only)"),
>  	OPT_END()
>  	};
>  
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 08/19] perf ftrace: add option -l/--long-info to show more info

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:17PM +0800, Changbin Du escreveu:
> Sometimes we want ftrace display more and longer information about trace.

Humm, -v? Or that would bring too much stuff from other parts of perf?
I guess so, perhaps as an option to the function-graph tracer, one that
combines, as you do, several options provided by that tracer?

- Arnaldo
 
> $ sudo perf ftrace -G -l
>  6800.190937 |   4)   <...>-7683   |   2.072 us    |  mutex_unlock();
>  6800.190941 |   4)   <...>-7683   |   2.171 us    |  __fsnotify_parent();
>  6800.190943 |   4)   <...>-7683   |   1.497 us    |  fsnotify();
>  6800.190944 |   4)   <...>-7683   |   0.775 us    |  __sb_end_write();
>  6800.190945 |   4)   <...>-7683   |   0.854 us    |  fpregs_assert_state_consistent();
>  6800.190947 |   4)   <...>-7683   |               |  do_syscall_64() {
>  6800.190948 |   4)   <...>-7683   |               |    __x64_sys_close() {
>  6800.190948 |   4)   <...>-7683   |               |      __close_fd() {
>  6800.190948 |   4)   <...>-7683   |   0.322 us    |        _raw_spin_lock();
>  6800.190949 |   4)   <...>-7683   |               |        filp_close() {
>  6800.190949 |   4)   <...>-7683   |   0.320 us    |          dnotify_flush();
>  6800.190950 |   4)   <...>-7683   |   0.325 us    |          locks_remove_posix();
> 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 28 ++++++++++++++++++++++++++++
>  1 file changed, 28 insertions(+)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index b16600a16efa..f11f2d3431b0 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -42,6 +42,7 @@ struct perf_ftrace {
>  	bool			func_stack_trace;
>  	bool			nosleep_time;
>  	bool			nofuncgraph_irqs;
> +	bool			long_info;
>  };
>  
>  struct filter_entry {
> @@ -190,6 +191,9 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
>  	write_tracing_option_file("func_stack_trace", "0");
>  	write_tracing_option_file("sleep-time", "1");
>  	write_tracing_option_file("funcgraph-irqs", "1");
> +	write_tracing_option_file("funcgraph-proc", "0");
> +	write_tracing_option_file("funcgraph-abstime", "0");
> +	write_tracing_option_file("irq-info", "0");
>  }
>  
>  static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
> @@ -371,6 +375,23 @@ static int set_tracing_funcgraph_irqs(struct perf_ftrace *ftrace)
>  	return 0;
>  }
>  
> +static int set_tracing_long_info(struct perf_ftrace *ftrace)
> +{
> +	if (!ftrace->long_info)
> +		return 0;
> +
> +	if (write_tracing_option_file("funcgraph-proc", "1") < 0)
> +		return -1;
> +
> +	if (write_tracing_option_file("funcgraph-abstime", "1") < 0)
> +		return -1;
> +
> +	if (write_tracing_option_file("irq-info", "1") < 0)
> +		return -1;
> +
> +	return 0;
> +}
> +
>  static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  {
>  	char *trace_file;
> @@ -449,6 +470,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  		goto out_reset;
>  	}
>  
> +	if (set_tracing_long_info(ftrace) < 0) {
> +		pr_err("failed to set tracing option funcgraph-proc/funcgraph-abstime/irq-info\n");
> +		goto out_reset;
> +	}
> +
>  	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
>  		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
>  		goto out_reset;
> @@ -588,6 +614,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		    "Measure on-CPU time only (function_graph only)"),
>  	OPT_BOOLEAN(0, "nofuncgraph-irqs", &ftrace.nofuncgraph_irqs,
>  		    "Ignore functions that happen inside interrupt (function_graph only)"),
> +	OPT_BOOLEAN('l', "long-info", &ftrace.long_info,
> +		    "Show process names, PIDs, timestamps, irq-info if available"),
>  	OPT_END()
>  	};
>  
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 09/19] perf ftrace: add support for trace option tracing_thresh

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:18PM +0800, Changbin Du escreveu:
> This adds an option '--tracing-thresh' to setup trace duration threshold
> for funcgraph tracer.

Ditto
 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 26 ++++++++++++++++++++++++++
>  1 file changed, 26 insertions(+)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index f11f2d3431b0..20bc14d6c5fb 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -43,6 +43,7 @@ struct perf_ftrace {
>  	bool			nosleep_time;
>  	bool			nofuncgraph_irqs;
>  	bool			long_info;
> +	unsigned		tracing_thresh;
>  };
>  
>  struct filter_entry {
> @@ -213,6 +214,9 @@ static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
>  	if (write_tracing_file("max_graph_depth", "0") < 0)
>  		return -1;
>  
> +	if (write_tracing_file("tracing_thresh", "0") < 0)
> +		return -1;
> +
>  	reset_tracing_filters();
>  	reset_tracing_options(ftrace);
>  	return 0;
> @@ -392,6 +396,21 @@ static int set_tracing_long_info(struct perf_ftrace *ftrace)
>  	return 0;
>  }
>  
> +static int set_tracing_thresh(struct perf_ftrace *ftrace)
> +{
> +	char buf[16];
> +
> +	if (ftrace->tracing_thresh == 0)
> +		return 0;
> +
> +	snprintf(buf, sizeof(buf), "%d", ftrace->tracing_thresh);
> +
> +	if (write_tracing_file("tracing_thresh", buf) < 0)
> +		return -1;
> +
> +	return 0;
> +}
> +
>  static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  {
>  	char *trace_file;
> @@ -475,6 +494,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  		goto out_reset;
>  	}
>  
> +	if (set_tracing_thresh(ftrace) < 0) {
> +		pr_err("failed to set tracing thresh\n");
> +		goto out_reset;
> +	}
> +
>  	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
>  		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
>  		goto out_reset;
> @@ -616,6 +640,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		    "Ignore functions that happen inside interrupt (function_graph only)"),
>  	OPT_BOOLEAN('l', "long-info", &ftrace.long_info,
>  		    "Show process names, PIDs, timestamps, irq-info if available"),
> +	OPT_UINTEGER(0, "tracing-thresh", &ftrace.tracing_thresh,
> +		     "Only show functions of which the duration is greater than <n>µs"),
>  	OPT_END()
>  	};
>  
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 10/19] perf ftrace: add support for trace option funcgraph-tail

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:19PM +0800, Changbin Du escreveu:
> This adds an option '--funcgraph-tail' for function graph tracer.

Ditto
 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 20 ++++++++++++++++++++
>  1 file changed, 20 insertions(+)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index 20bc14d6c5fb..2ef5d1c4b23c 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -42,6 +42,7 @@ struct perf_ftrace {
>  	bool			func_stack_trace;
>  	bool			nosleep_time;
>  	bool			nofuncgraph_irqs;
> +	bool			funcgraph_tail;
>  	bool			long_info;
>  	unsigned		tracing_thresh;
>  };
> @@ -192,6 +193,7 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
>  	write_tracing_option_file("func_stack_trace", "0");
>  	write_tracing_option_file("sleep-time", "1");
>  	write_tracing_option_file("funcgraph-irqs", "1");
> +	write_tracing_option_file("funcgraph-tail", "0");
>  	write_tracing_option_file("funcgraph-proc", "0");
>  	write_tracing_option_file("funcgraph-abstime", "0");
>  	write_tracing_option_file("irq-info", "0");
> @@ -411,6 +413,17 @@ static int set_tracing_thresh(struct perf_ftrace *ftrace)
>  	return 0;
>  }
>  
> +static int set_tracing_funcgraph_tail(struct perf_ftrace *ftrace)
> +{
> +	if (!ftrace->funcgraph_tail)
> +		return 0;
> +
> +	if (write_tracing_option_file("funcgraph-tail", "1") < 0)
> +		return -1;
> +
> +	return 0;
> +}
> +
>  static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  {
>  	char *trace_file;
> @@ -499,6 +512,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  		goto out_reset;
>  	}
>  
> +	if (set_tracing_funcgraph_tail(ftrace) < 0) {
> +		pr_err("failed to set tracing option funcgraph-tail\n");
> +		goto out_reset;
> +	}
> +
>  	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
>  		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
>  		goto out_reset;
> @@ -638,6 +656,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		    "Measure on-CPU time only (function_graph only)"),
>  	OPT_BOOLEAN(0, "nofuncgraph-irqs", &ftrace.nofuncgraph_irqs,
>  		    "Ignore functions that happen inside interrupt (function_graph only)"),
> +	OPT_BOOLEAN(0, "funcgraph-tail", &ftrace.funcgraph_tail,
> +		    "Show function tails comment (function_graph only)"),
>  	OPT_BOOLEAN('l', "long-info", &ftrace.long_info,
>  		    "Show process names, PIDs, timestamps, irq-info if available"),
>  	OPT_UINTEGER(0, "tracing-thresh", &ftrace.tracing_thresh,
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 10/19] perf ftrace: add support for trace option funcgraph-tail

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:19PM +0800, Changbin Du escreveu:
> This adds an option '--funcgraph-tail' for function graph tracer.

And I think we should make these available in a compact way, as Intel PT
has, i.e. instead of doing something like:

   --function-graph-options nosleep_time,noirqs,no_tail,long_info

We could have:

   -G ns,ni,nt,li

To save on typing, or something like that.

- Arnaldo
 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 20 ++++++++++++++++++++
>  1 file changed, 20 insertions(+)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index 20bc14d6c5fb..2ef5d1c4b23c 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -42,6 +42,7 @@ struct perf_ftrace {
>  	bool			func_stack_trace;
>  	bool			nosleep_time;
>  	bool			nofuncgraph_irqs;
> +	bool			funcgraph_tail;
>  	bool			long_info;
>  	unsigned		tracing_thresh;
>  };
> @@ -192,6 +193,7 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
>  	write_tracing_option_file("func_stack_trace", "0");
>  	write_tracing_option_file("sleep-time", "1");
>  	write_tracing_option_file("funcgraph-irqs", "1");
> +	write_tracing_option_file("funcgraph-tail", "0");
>  	write_tracing_option_file("funcgraph-proc", "0");
>  	write_tracing_option_file("funcgraph-abstime", "0");
>  	write_tracing_option_file("irq-info", "0");
> @@ -411,6 +413,17 @@ static int set_tracing_thresh(struct perf_ftrace *ftrace)
>  	return 0;
>  }
>  
> +static int set_tracing_funcgraph_tail(struct perf_ftrace *ftrace)
> +{
> +	if (!ftrace->funcgraph_tail)
> +		return 0;
> +
> +	if (write_tracing_option_file("funcgraph-tail", "1") < 0)
> +		return -1;
> +
> +	return 0;
> +}
> +
>  static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  {
>  	char *trace_file;
> @@ -499,6 +512,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  		goto out_reset;
>  	}
>  
> +	if (set_tracing_funcgraph_tail(ftrace) < 0) {
> +		pr_err("failed to set tracing option funcgraph-tail\n");
> +		goto out_reset;
> +	}
> +
>  	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
>  		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
>  		goto out_reset;
> @@ -638,6 +656,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		    "Measure on-CPU time only (function_graph only)"),
>  	OPT_BOOLEAN(0, "nofuncgraph-irqs", &ftrace.nofuncgraph_irqs,
>  		    "Ignore functions that happen inside interrupt (function_graph only)"),
> +	OPT_BOOLEAN(0, "funcgraph-tail", &ftrace.funcgraph_tail,
> +		    "Show function tails comment (function_graph only)"),
>  	OPT_BOOLEAN('l', "long-info", &ftrace.long_info,
>  		    "Show process names, PIDs, timestamps, irq-info if available"),
>  	OPT_UINTEGER(0, "tracing-thresh", &ftrace.tracing_thresh,
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 11/19] perf ftrace: add option '-u/--userstacktrace' to show userspace stacktrace

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:20PM +0800, Changbin Du escreveu:
> This adds an option ''-u/--userstacktrace' for function tracer to display
> userspace back trace.

Probably we should have this as a term, an option to --call-graph?

For --call-graph the way to suppress this is to ask for the event to be
in the kernel only, i.e. something like:

	perf record -e cycles:k --call-graph

So perhaps we should have something like:

	perf ftrace --call-graph 

With some default, possibly a bit different than the other perf tools,
just including the kernel, and accepting:

	perf ftrace --call-graph
	perf ftrace --call-graph k
	perf ftrace --call-graph u
	perf ftrace --call-graph uk

- Arnaldo
 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 24 ++++++++++++++++++++++++
>  1 file changed, 24 insertions(+)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index 2ef5d1c4b23c..ab76ba66bd9e 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -40,6 +40,7 @@ struct perf_ftrace {
>  	struct list_head	nograph_funcs;
>  	int			graph_depth;
>  	bool			func_stack_trace;
> +	bool			userstacktrace;
>  	bool			nosleep_time;
>  	bool			nofuncgraph_irqs;
>  	bool			funcgraph_tail;
> @@ -197,6 +198,8 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
>  	write_tracing_option_file("funcgraph-proc", "0");
>  	write_tracing_option_file("funcgraph-abstime", "0");
>  	write_tracing_option_file("irq-info", "0");
> +	write_tracing_option_file("userstacktrace", "0");
> +	write_tracing_option_file("sym-userobj", "0");
>  }
>  
>  static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
> @@ -287,6 +290,20 @@ static int set_tracing_func_stack_trace(struct perf_ftrace *ftrace)
>  	return 0;
>  }
>  
> +static int set_tracing_userstacktrace(struct perf_ftrace *ftrace)
> +{
> +	if (!ftrace->userstacktrace)
> +		return 0;
> +
> +	if (write_tracing_option_file("userstacktrace", "1") < 0)
> +		return -1;
> +
> +	if (write_tracing_option_file("sym-userobj", "1") < 0)
> +		return -1;
> +
> +	return 0;
> +}
> +
>  static int reset_tracing_cpu(void)
>  {
>  	struct perf_cpu_map *cpumap = perf_cpu_map__new(NULL);
> @@ -482,6 +499,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  		goto out_reset;
>  	}
>  
> +	if (set_tracing_userstacktrace(ftrace) < 0) {
> +		pr_err("failed to set tracing option userstacktrace\n");
> +		goto out_reset;
> +	}
> +
>  	if (set_tracing_filters(ftrace) < 0) {
>  		pr_err("failed to set tracing filters\n");
>  		goto out_reset;
> @@ -644,6 +666,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		     "do not trace given functions", parse_filter_func),
>  	OPT_BOOLEAN('s', "func-stack-trace", &ftrace.func_stack_trace,
>  		    "Show kernel stack trace for function tracer"),
> +	OPT_BOOLEAN('u', "userstacktrace", &ftrace.userstacktrace,
> +		    "Show stacktrace of the current user space thread"),
>  	OPT_CALLBACK_DEFAULT('G', "graph-funcs", &ftrace.graph_funcs, "func",
>  		     "Set graph filter on given functions (imply to use function_graph tracer)",
>  		     parse_filter_func, "*"),
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 12/19] perf ftrace: add support for tracing children processes

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:21PM +0800, Changbin Du escreveu:
> This adds an option '--trace-children' to allow us trace children
> processes spawned by our target.

--inherit

- Arnaldo
 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 20 ++++++++++++++++++++
>  1 file changed, 20 insertions(+)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index ab76ba66bd9e..8fd95c109fe8 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -46,6 +46,7 @@ struct perf_ftrace {
>  	bool			funcgraph_tail;
>  	bool			long_info;
>  	unsigned		tracing_thresh;
> +	bool			trace_children;
>  };
>  
>  struct filter_entry {
> @@ -200,6 +201,7 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
>  	write_tracing_option_file("irq-info", "0");
>  	write_tracing_option_file("userstacktrace", "0");
>  	write_tracing_option_file("sym-userobj", "0");
> +	write_tracing_option_file("function-fork", "0");
>  }
>  
>  static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
> @@ -441,6 +443,17 @@ static int set_tracing_funcgraph_tail(struct perf_ftrace *ftrace)
>  	return 0;
>  }
>  
> +static int set_tracing_trace_children(struct perf_ftrace *ftrace)
> +{
> +	if (!ftrace->trace_children)
> +		return 0;
> +
> +	if (write_tracing_option_file("function-fork", "1") < 0)
> +		return -1;
> +
> +	return 0;
> +}
> +
>  static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  {
>  	char *trace_file;
> @@ -539,6 +552,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  		goto out_reset;
>  	}
>  
> +	if (set_tracing_trace_children(ftrace) < 0) {
> +		pr_err("failed to set tracing option function-fork\n");
> +		goto out_reset;
> +	}
> +
>  	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
>  		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
>  		goto out_reset;
> @@ -686,6 +704,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		    "Show process names, PIDs, timestamps, irq-info if available"),
>  	OPT_UINTEGER(0, "tracing-thresh", &ftrace.tracing_thresh,
>  		     "Only show functions of which the duration is greater than <n>µs"),
> +	OPT_BOOLEAN(0, "trace-children", &ftrace.trace_children,
> +		    "Trace children processes"),
>  	OPT_END()
>  	};
>  
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 13/19] perf ftrace: add option '-b/--buffer-size' to set per-cpu buffer size

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:22PM +0800, Changbin Du escreveu:
> This adds an option '-b/--buffer-size' to allow us set the size of per-cpu
> tracing buffer.

-m
 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 45 ++++++++++++++++++++++++++++---------
>  1 file changed, 35 insertions(+), 10 deletions(-)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index 8fd95c109fe8..a93fbdac6aa4 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -47,6 +47,7 @@ struct perf_ftrace {
>  	bool			long_info;
>  	unsigned		tracing_thresh;
>  	bool			trace_children;
> +	unsigned		buffer_size_kb;
>  };
>  
>  struct filter_entry {
> @@ -187,6 +188,17 @@ static int read_tracing_file_to_stdout(const char *name)
>  	return ret;
>  }
>  
> +static int write_tracing_file_int(const char *name, int value)
> +{
> +	char buf[16];
> +
> +	snprintf(buf, sizeof(buf), "%d", value);
> +	if (write_tracing_file(name, buf) < 0)
> +		return -1;
> +
> +	return 0;
> +}
> +
>  static int reset_tracing_cpu(void);
>  static void reset_tracing_filters(void);
>  
> @@ -360,8 +372,6 @@ static void reset_tracing_filters(void)
>  
>  static int set_tracing_depth(struct perf_ftrace *ftrace)
>  {
> -	char buf[16];
> -
>  	if (ftrace->graph_depth == 0)
>  		return 0;
>  
> @@ -370,9 +380,7 @@ static int set_tracing_depth(struct perf_ftrace *ftrace)
>  		return -1;
>  	}
>  
> -	snprintf(buf, sizeof(buf), "%d", ftrace->graph_depth);
> -
> -	if (write_tracing_file("max_graph_depth", buf) < 0)
> +	if (write_tracing_file_int("max_graph_depth", ftrace->graph_depth) < 0)
>  		return -1;
>  
>  	return 0;
> @@ -419,14 +427,10 @@ static int set_tracing_long_info(struct perf_ftrace *ftrace)
>  
>  static int set_tracing_thresh(struct perf_ftrace *ftrace)
>  {
> -	char buf[16];
> -
>  	if (ftrace->tracing_thresh == 0)
>  		return 0;
>  
> -	snprintf(buf, sizeof(buf), "%d", ftrace->tracing_thresh);
> -
> -	if (write_tracing_file("tracing_thresh", buf) < 0)
> +	if (write_tracing_file_int("tracing_thresh", ftrace->tracing_thresh) < 0)
>  		return -1;
>  
>  	return 0;
> @@ -454,6 +458,20 @@ static int set_tracing_trace_children(struct perf_ftrace *ftrace)
>  	return 0;
>  }
>  
> +static int set_tracing_buffer_size_kb(struct perf_ftrace *ftrace)
> +{
> +	int ret;
> +
> +	if (ftrace->buffer_size_kb == 0)
> +		return 0;
> +
> +	ret = write_tracing_file_int("buffer_size_kb", ftrace->buffer_size_kb);
> +	if (ret < 0)
> +		return ret;
> +
> +	return 0;
> +}
> +
>  static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  {
>  	char *trace_file;
> @@ -557,6 +575,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  		goto out_reset;
>  	}
>  
> +	if (set_tracing_buffer_size_kb(ftrace) < 0) {
> +		pr_err("failed to set tracing per-cpu buffer size\n");
> +		goto out_reset;
> +	}
> +
>  	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
>  		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
>  		goto out_reset;
> @@ -706,6 +729,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		     "Only show functions of which the duration is greater than <n>µs"),
>  	OPT_BOOLEAN(0, "trace-children", &ftrace.trace_children,
>  		    "Trace children processes"),
> +	OPT_UINTEGER('b', "buffer-size", &ftrace.buffer_size_kb,
> +		     "size of per cpu buffer in kb"),
>  	OPT_END()
>  	};
>  
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 14/19] perf ftrace: add option -P/--no-pager to disable pager

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:23PM +0800, Changbin Du escreveu:
> Sometimes we want perf displays trace immediately. So this adds an option
> '-P/--no-pager' to disable pager if needed.


Try:

perf record sleep 5
perf script

Then:

perf --no-pager script

I.e. its there already.

- Arnaldo

> 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 17 ++++++++++++++---
>  1 file changed, 14 insertions(+), 3 deletions(-)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index a93fbdac6aa4..64c22f367ba2 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -48,6 +48,7 @@ struct perf_ftrace {
>  	unsigned		tracing_thresh;
>  	bool			trace_children;
>  	unsigned		buffer_size_kb;
> +	bool			no_pager;
>  };
>  
>  struct filter_entry {
> @@ -56,6 +57,7 @@ struct filter_entry {
>  };
>  
>  static volatile int workload_exec_errno;
> +static bool interrupted;
>  static bool done;
>  
>  static void sig_handler(int sig __maybe_unused)
> @@ -63,6 +65,12 @@ static void sig_handler(int sig __maybe_unused)
>  	done = true;
>  }
>  
> +static void sig_handler_int(int sig __maybe_unused)
> +{
> +	sig_handler(sig);
> +	interrupted = 1;
> +}
> +
>  /*
>   * perf_evlist__prepare_workload will send a SIGUSR1 if the fork fails, since
>   * we asked by setting its exec_error to the function below,
> @@ -492,7 +500,7 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  		return -1;
>  	}
>  
> -	signal(SIGINT, sig_handler);
> +	signal(SIGINT, sig_handler_int);
>  	signal(SIGUSR1, sig_handler);
>  	signal(SIGCHLD, sig_handler);
>  	signal(SIGPIPE, sig_handler);
> @@ -585,7 +593,8 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  		goto out_reset;
>  	}
>  
> -	setup_pager();
> +	if (!ftrace->no_pager)
> +		setup_pager();
>  
>  	trace_file = get_tracing_file("trace_pipe");
>  	if (!trace_file) {
> @@ -636,7 +645,7 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  	}
>  
>  	/* read remaining buffer contents */
> -	while (true) {
> +	while (true && !interrupted) {
>  		int n = read(trace_fd, buf, sizeof(buf));
>  		if (n <= 0)
>  			break;
> @@ -731,6 +740,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		    "Trace children processes"),
>  	OPT_UINTEGER('b', "buffer-size", &ftrace.buffer_size_kb,
>  		     "size of per cpu buffer in kb"),
> +	OPT_BOOLEAN('P', "no-pager", &ftrace.no_pager,
> +		    "Do not use pager"),
>  	OPT_END()
>  	};
>  
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 15/19] perf ftrace: show trace column header

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:24PM +0800, Changbin Du escreveu:
> This makes perf-ftrace display column header before printing trace.

[acme@five perf]$ perf report -h header

 Usage: perf report [<options>]

        --header          Show data header.
        --header-only     Show only data header.

[acme@five perf]$

Perhaps there is value in --header-only to show the default that will be
setup when using some set of options.

- Arnaldo

> $ sudo perf ftrace
> \# tracer: function
> \#
> \# entries-in-buffer/entries-written: 0/0   #P:8
> \#
> \#           TASK-PID     CPU#   TIMESTAMP  FUNCTION
> \#              | |         |       |         |
>            <...>-9246  [006]  10726.262760: mutex_unlock <-rb_simple_write
>            <...>-9246  [006]  10726.262764: __fsnotify_parent <-vfs_write
>            <...>-9246  [006]  10726.262765: fsnotify <-vfs_write
>            <...>-9246  [006]  10726.262766: __sb_end_write <-vfs_write
>            <...>-9246  [006]  10726.262767: fpregs_assert_state_consistent <-do_syscall_64
> 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 3 +++
>  1 file changed, 3 insertions(+)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index 64c22f367ba2..0b39b6a88026 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -614,6 +614,9 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  	fcntl(trace_fd, F_SETFL, O_NONBLOCK);
>  	pollfd.fd = trace_fd;
>  
> +	/* display column headers */
> +	read_tracing_file_to_stdout("trace");
> +
>  	if (write_tracing_file("tracing_on", "1") < 0) {
>  		pr_err("can't enable tracing\n");
>  		goto out_close_fd;
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 16/19] perf ftrace: add option -t/--tid to filter by thread id

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:25PM +0800, Changbin Du escreveu:
> This allows us to trace single thread instead of the whole process.

I was going to adjust the patch to add, but you forgot to add the entry
to tools/perf/Documentation/perf-ftrace.txt,

Please do so,

- Arnaldo
 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 2 ++
>  1 file changed, 2 insertions(+)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index 0b39b6a88026..8d04e5afe2d3 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -707,6 +707,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		    "List available functions to filter"),
>  	OPT_STRING('p', "pid", &ftrace.target.pid, "pid",
>  		   "trace on existing process id"),
> +	OPT_STRING('t', "tid", &ftrace.target.tid, "tid",
> +		   "trace on existing thread id (exclusive to --pid)"),
>  	OPT_INCR('v', "verbose", &verbose,
>  		 "be more verbose"),
>  	OPT_BOOLEAN('a', "all-cpus", &ftrace.target.system_wide,
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 17/19] perf ftrace: add option -d/--delay to delay tracing

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:26PM +0800, Changbin Du escreveu:
> This adds an option '-d/--delay' to allow us to start tracing some
> times later after workload is launched.

[acme@five perf]$ perf record -h delay

 Usage: perf record [<options>] [<command>]
    or: perf record [<options>] -- <command> [<options>]

    -D, --delay <n>       ms to wait before starting measurement after program start

[acme@five perf]$
 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 19 ++++++++++++++++---
>  1 file changed, 16 insertions(+), 3 deletions(-)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index 8d04e5afe2d3..d376b37c53fc 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -49,6 +49,7 @@ struct perf_ftrace {
>  	bool			trace_children;
>  	unsigned		buffer_size_kb;
>  	bool			no_pager;
> +	unsigned		initial_delay;
>  };
>  
>  struct filter_entry {
> @@ -617,13 +618,23 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  	/* display column headers */
>  	read_tracing_file_to_stdout("trace");
>  
> -	if (write_tracing_file("tracing_on", "1") < 0) {
> -		pr_err("can't enable tracing\n");
> -		goto out_close_fd;
> +	if (!ftrace->initial_delay) {
> +		if (write_tracing_file("tracing_on", "1") < 0) {
> +			pr_err("can't enable tracing\n");
> +			goto out_close_fd;
> +		}
>  	}
>  
>  	perf_evlist__start_workload(ftrace->evlist);
>  
> +	if (ftrace->initial_delay) {
> +		usleep(ftrace->initial_delay * 1000);
> +		if (write_tracing_file("tracing_on", "1") < 0) {
> +			pr_err("can't enable tracing\n");
> +			goto out_close_fd;
> +		}
> +	}
> +
>  	while (!done) {
>  		if (poll(&pollfd, 1, -1) < 0)
>  			break;
> @@ -747,6 +758,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		     "size of per cpu buffer in kb"),
>  	OPT_BOOLEAN('P', "no-pager", &ftrace.no_pager,
>  		    "Do not use pager"),
> +	OPT_UINTEGER('d', "delay", &ftrace.initial_delay,
> +		     "Wait <n> ms before tracing"),
>  	OPT_END()
>  	};
>  
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 18/19] perf ftrace: add option --latency-format to display more info about delay

[Arnaldo Carvalho de Melo]

Em Sun, May 10, 2020 at 11:06:27PM +0800, Changbin Du escreveu:
> This is for the function graph tracer to display more info about latency.
> The execution context is shown in this case.

Ok, another --function-graph-opts entry, maybe you can just make -G
receive optional args.

- Arnaldo
 
> $ sudo perf ftrace -G --latency-format
> \# tracer: function_graph
> \#
>  1)  .... |   0.992 us    |  mutex_unlock();
>  1)  .... |   1.404 us    |  __fsnotify_parent();
>  1)  .... |   1.023 us    |  fsnotify();
>  1)  .... |   0.335 us    |  __sb_end_write();
>  1)  d... |   0.439 us    |  fpregs_assert_state_consistent();
>  1)  d... |               |  do_syscall_64() {
>  1)  .... |               |    __x64_sys_close() {
> 
> Signed-off-by: Changbin Du <changbin.du@gmail.com>
> ---
>  tools/perf/builtin-ftrace.c | 20 ++++++++++++++++++++
>  1 file changed, 20 insertions(+)
> 
> diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> index d376b37c53fc..fd8e2f305136 100644
> --- a/tools/perf/builtin-ftrace.c
> +++ b/tools/perf/builtin-ftrace.c
> @@ -44,6 +44,7 @@ struct perf_ftrace {
>  	bool			nosleep_time;
>  	bool			nofuncgraph_irqs;
>  	bool			funcgraph_tail;
> +	bool			latency_format;
>  	bool			long_info;
>  	unsigned		tracing_thresh;
>  	bool			trace_children;
> @@ -217,6 +218,7 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
>  	write_tracing_option_file("sleep-time", "1");
>  	write_tracing_option_file("funcgraph-irqs", "1");
>  	write_tracing_option_file("funcgraph-tail", "0");
> +	write_tracing_option_file("latency-format", "0");
>  	write_tracing_option_file("funcgraph-proc", "0");
>  	write_tracing_option_file("funcgraph-abstime", "0");
>  	write_tracing_option_file("irq-info", "0");
> @@ -456,6 +458,17 @@ static int set_tracing_funcgraph_tail(struct perf_ftrace *ftrace)
>  	return 0;
>  }
>  
> +static int set_tracing_latency_format(struct perf_ftrace *ftrace)
> +{
> +	if (!ftrace->latency_format)
> +		return 0;
> +
> +	if (write_tracing_option_file("latency-format", "1") < 0)
> +		return -1;
> +
> +	return 0;
> +}
> +
>  static int set_tracing_trace_children(struct perf_ftrace *ftrace)
>  {
>  	if (!ftrace->trace_children)
> @@ -579,6 +592,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
>  		goto out_reset;
>  	}
>  
> +	if (set_tracing_latency_format(ftrace) < 0) {
> +		pr_err("failed to set tracing option latency-format\n");
> +		goto out_reset;
> +	}
> +
>  	if (set_tracing_trace_children(ftrace) < 0) {
>  		pr_err("failed to set tracing option function-fork\n");
>  		goto out_reset;
> @@ -748,6 +766,8 @@ int cmd_ftrace(int argc, const char **argv)
>  		    "Ignore functions that happen inside interrupt (function_graph only)"),
>  	OPT_BOOLEAN(0, "funcgraph-tail", &ftrace.funcgraph_tail,
>  		    "Show function tails comment (function_graph only)"),
> +	OPT_BOOLEAN(0, "latency-format", &ftrace.latency_format,
> +		    "displays additional information about the latency (function_graph only)"),
>  	OPT_BOOLEAN('l', "long-info", &ftrace.long_info,
>  		    "Show process names, PIDs, timestamps, irq-info if available"),
>  	OPT_UINTEGER(0, "tracing-thresh", &ftrace.tracing_thresh,
> -- 
> 2.25.1
> 

-- 

- Arnaldo

Re: [PATCH 00/19] perf: ftrace enhancement

[Changbin Du]

On Wed, May 20, 2020 at 05:31:28PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Sun, May 10, 2020 at 11:06:09PM +0800, Changbin Du escreveu:
> > The perf has basic kernel ftrace support but lack support of most tracing
> > options. This serias is target to enhance the perf ftrace functionality so
> > that we can make full use of kernel ftrace with only perf.
> 
> Quite a lot! Going thru it now.
> 
> - Arnaldo
>  
Hi Arnaldo, thanks for your detailed review. Sorry for slow response since I am
just getting busy with my own affairs. I'll do the changes acoording
to your comments one by one as soon as possilbe.

[snip..]
-- 
Cheers,
Changbin Du

Re: [PATCH 03/19] perf ftrace: select function/function_graph tracer automatically

[Namhyung Kim]

Hello,

On Mon, May 11, 2020 at 12:07 AM Changbin Du <changbin.du@gmail.com> wrote:
>
> The '-g/-G' options have already implied function_graph tracer should be
> used instead of function tracer. So the extra option '--tracer' can be
> killed.
>
> This patch changes the behavior as below:
>   - By default, function tracer is used.

Why?  You also removed -t/--tracer option.

>   - If '-g' or '-G' option is on, then function_graph tracer is used.

I'm ok with this.

>   - The perf configuration item 'ftrace.tracer' is removed.

Why?

>   - The default filter for -G is to trace all functions.

There's no reason to use -G option then.  Also It might be confusing
if it sees an argument - whether it's a function or a workload.

I just don't know why you want to change this as we have a way to
change the default tracer already.

Thanks

Namhyung

Re: [PATCH 06/19] perf ftrace: add support for trace option sleep-time

[Namhyung Kim]

On Thu, May 21, 2020 at 6:01 AM Arnaldo Carvalho de Melo
<acme@kernel.org> wrote:
>
> Em Sun, May 10, 2020 at 11:06:15PM +0800, Changbin Du escreveu:
> > This adds an option '--nosleep-time' which allow us only to measure
> > on-CPU time. This option is function_graph tracer only.
>
> This seems, for now, very specific to the function_graph tracer, so
> perhaps we should have a:
>
>         --function_graph_opts nosleep-time,other,another,etc
>
> ?

Agreed.  Also I don't want to add an option in a negative form
as it's confusing.  Actually, our option parser can recognize
--no-xxx form automatically so adding a positive option (--xxx)
can handle that too.

Thanks
Namhyung

Re: [PATCH 08/19] perf ftrace: add option -l/--long-info to show more info

[Namhyung Kim]

On Thu, May 21, 2020 at 6:03 AM Arnaldo Carvalho de Melo
<acme@kernel.org> wrote:
>
> Em Sun, May 10, 2020 at 11:06:17PM +0800, Changbin Du escreveu:
> > Sometimes we want ftrace display more and longer information about trace.
>
> Humm, -v? Or that would bring too much stuff from other parts of perf?
> I guess so, perhaps as an option to the function-graph tracer, one that
> combines, as you do, several options provided by that tracer?

I think -v option is to debug perf tool's behavior while this is to change
the output (or behavior).

Thanks,
Namhyung

Re: [PATCH 10/19] perf ftrace: add support for trace option funcgraph-tail

[Namhyung Kim]

On Thu, May 21, 2020 at 6:05 AM Arnaldo Carvalho de Melo
<acme@kernel.org> wrote:
>
> Em Sun, May 10, 2020 at 11:06:19PM +0800, Changbin Du escreveu:
> > This adds an option '--funcgraph-tail' for function graph tracer.
>
> And I think we should make these available in a compact way, as Intel PT
> has, i.e. instead of doing something like:
>
>    --function-graph-options nosleep_time,noirqs,no_tail,long_info
>
> We could have:
>
>    -G ns,ni,nt,li
>
> To save on typing, or something like that.

Looks good.  What about separate 'no' prefix from two letter alias?

st, nost, ir, noir, ...

Thanks
Namhyung

Re: [PATCH 08/19] perf ftrace: add option -l/--long-info to show more info

[Changbin Du]

On Wed, May 20, 2020 at 06:02:57PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Sun, May 10, 2020 at 11:06:17PM +0800, Changbin Du escreveu:
> > Sometimes we want ftrace display more and longer information about trace.
> 
> Humm, -v? Or that would bring too much stuff from other parts of perf?
> I guess so, perhaps as an option to the function-graph tracer, one that
> combines, as you do, several options provided by that tracer?
>
yes, this option enables mutiple trace options onetime, which will make the
function tracer or graph tracer output as much as more information per-line. So I call it
as 'long'.

> - Arnaldo
>  
> > $ sudo perf ftrace -G -l
> >  6800.190937 |   4)   <...>-7683   |   2.072 us    |  mutex_unlock();
> >  6800.190941 |   4)   <...>-7683   |   2.171 us    |  __fsnotify_parent();
> >  6800.190943 |   4)   <...>-7683   |   1.497 us    |  fsnotify();
> >  6800.190944 |   4)   <...>-7683   |   0.775 us    |  __sb_end_write();
> >  6800.190945 |   4)   <...>-7683   |   0.854 us    |  fpregs_assert_state_consistent();
> >  6800.190947 |   4)   <...>-7683   |               |  do_syscall_64() {
> >  6800.190948 |   4)   <...>-7683   |               |    __x64_sys_close() {
> >  6800.190948 |   4)   <...>-7683   |               |      __close_fd() {
> >  6800.190948 |   4)   <...>-7683   |   0.322 us    |        _raw_spin_lock();
> >  6800.190949 |   4)   <...>-7683   |               |        filp_close() {
> >  6800.190949 |   4)   <...>-7683   |   0.320 us    |          dnotify_flush();
> >  6800.190950 |   4)   <...>-7683   |   0.325 us    |          locks_remove_posix();
> > 
> > Signed-off-by: Changbin Du <changbin.du@gmail.com>
> > ---
> >  tools/perf/builtin-ftrace.c | 28 ++++++++++++++++++++++++++++
> >  1 file changed, 28 insertions(+)
> > 
> > diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> > index b16600a16efa..f11f2d3431b0 100644
> > --- a/tools/perf/builtin-ftrace.c
> > +++ b/tools/perf/builtin-ftrace.c
> > @@ -42,6 +42,7 @@ struct perf_ftrace {
> >  	bool			func_stack_trace;
> >  	bool			nosleep_time;
> >  	bool			nofuncgraph_irqs;
> > +	bool			long_info;
> >  };
> >  
> >  struct filter_entry {
> > @@ -190,6 +191,9 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
> >  	write_tracing_option_file("func_stack_trace", "0");
> >  	write_tracing_option_file("sleep-time", "1");
> >  	write_tracing_option_file("funcgraph-irqs", "1");
> > +	write_tracing_option_file("funcgraph-proc", "0");
> > +	write_tracing_option_file("funcgraph-abstime", "0");
> > +	write_tracing_option_file("irq-info", "0");
> >  }
> >  
> >  static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
> > @@ -371,6 +375,23 @@ static int set_tracing_funcgraph_irqs(struct perf_ftrace *ftrace)
> >  	return 0;
> >  }
> >  
> > +static int set_tracing_long_info(struct perf_ftrace *ftrace)
> > +{
> > +	if (!ftrace->long_info)
> > +		return 0;
> > +
> > +	if (write_tracing_option_file("funcgraph-proc", "1") < 0)
> > +		return -1;
> > +
> > +	if (write_tracing_option_file("funcgraph-abstime", "1") < 0)
> > +		return -1;
> > +
> > +	if (write_tracing_option_file("irq-info", "1") < 0)
> > +		return -1;
> > +
> > +	return 0;
> > +}
> > +
> >  static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
> >  {
> >  	char *trace_file;
> > @@ -449,6 +470,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
> >  		goto out_reset;
> >  	}
> >  
> > +	if (set_tracing_long_info(ftrace) < 0) {
> > +		pr_err("failed to set tracing option funcgraph-proc/funcgraph-abstime/irq-info\n");
> > +		goto out_reset;
> > +	}
> > +
> >  	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
> >  		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
> >  		goto out_reset;
> > @@ -588,6 +614,8 @@ int cmd_ftrace(int argc, const char **argv)
> >  		    "Measure on-CPU time only (function_graph only)"),
> >  	OPT_BOOLEAN(0, "nofuncgraph-irqs", &ftrace.nofuncgraph_irqs,
> >  		    "Ignore functions that happen inside interrupt (function_graph only)"),
> > +	OPT_BOOLEAN('l', "long-info", &ftrace.long_info,
> > +		    "Show process names, PIDs, timestamps, irq-info if available"),
> >  	OPT_END()
> >  	};
> >  
> > -- 
> > 2.25.1
> > 
> 
> -- 
> 
> - Arnaldo

-- 
Cheers,
Changbin Du

Re: [PATCH 06/19] perf ftrace: add support for trace option sleep-time

[Changbin Du]

On Sun, May 31, 2020 at 02:56:41PM +0900, Namhyung Kim wrote:
> On Thu, May 21, 2020 at 6:01 AM Arnaldo Carvalho de Melo
> <acme@kernel.org> wrote:
> >
> > Em Sun, May 10, 2020 at 11:06:15PM +0800, Changbin Du escreveu:
> > > This adds an option '--nosleep-time' which allow us only to measure
> > > on-CPU time. This option is function_graph tracer only.
> >
> > This seems, for now, very specific to the function_graph tracer, so
> > perhaps we should have a:
> >
> >         --function_graph_opts nosleep-time,other,another,etc
> >
> > ?
> 
> Agreed.  Also I don't want to add an option in a negative form
> as it's confusing.  Actually, our option parser can recognize
> --no-xxx form automatically so adding a positive option (--xxx)
> can handle that too.
>
Agree. And for the name, maybe we can align to original trace options.

root@WRT-WX9:/sys/kernel/debug/tracing# cat trace_options
print-parent
nosym-offset
nosym-addr
noverbose
noraw
nohex
nobin
noblock
trace_printk
annotate
nouserstacktrace
nosym-userobj
noprintk-msg-only
context-info
nolatency-format
record-cmd
norecord-tgid
overwrite
nodisable_on_free
irq-info
markers
noevent-fork
function-trace
nofunction-fork
nodisplay-graph
nostacktrace
notest_nop_accept
notest_nop_refuse


> Thanks
> Namhyung

-- 
Cheers,
Changbin Du

Re: [PATCH 03/19] perf ftrace: select function/function_graph tracer automatically

[Changbin Du]

On Sun, May 31, 2020 at 02:52:23PM +0900, Namhyung Kim wrote:
> Hello,
> 
> On Mon, May 11, 2020 at 12:07 AM Changbin Du <changbin.du@gmail.com> wrote:
> >
> > The '-g/-G' options have already implied function_graph tracer should be
> > used instead of function tracer. So the extra option '--tracer' can be
> > killed.
> >
> > This patch changes the behavior as below:
> >   - By default, function tracer is used.
> 
> Why?  You also removed -t/--tracer option.
>
To save typing. By this way, we do not need the -t/--tracer option which is redundant.
When -G is given, it means graph tracer should be used. If not, function tracer.

> >   - If '-g' or '-G' option is on, then function_graph tracer is used.
> 
> I'm ok with this.
> 
> >   - The perf configuration item 'ftrace.tracer' is removed.
> 
> Why?
> 
As explained above, we don't need this option any longer.

> >   - The default filter for -G is to trace all functions.
> 
> There's no reason to use -G option then.  Also It might be confusing
> if it sees an argument - whether it's a function or a workload.
> 
> I just don't know why you want to change this as we have a way to
> change the default tracer already.
> 
> Thanks
> 
> Namhyung

-- 
Cheers,
Changbin Du

Re: [PATCH 17/19] perf ftrace: add option -d/--delay to delay tracing

[Changbin Du]

On Wed, May 20, 2020 at 06:13:31PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Sun, May 10, 2020 at 11:06:26PM +0800, Changbin Du escreveu:
> > This adds an option '-d/--delay' to allow us to start tracing some
> > times later after workload is launched.
> 
> [acme@five perf]$ perf record -h delay
> 
>  Usage: perf record [<options>] [<command>]
>     or: perf record [<options>] -- <command> [<options>]
> 
>     -D, --delay <n>       ms to wait before starting measurement after program start
> 
> [acme@five perf]$
>
Sure, will change the short option name.

> > Signed-off-by: Changbin Du <changbin.du@gmail.com>
> > ---
> >  tools/perf/builtin-ftrace.c | 19 ++++++++++++++++---
> >  1 file changed, 16 insertions(+), 3 deletions(-)
> > 
> > diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> > index 8d04e5afe2d3..d376b37c53fc 100644
> > --- a/tools/perf/builtin-ftrace.c
> > +++ b/tools/perf/builtin-ftrace.c
> > @@ -49,6 +49,7 @@ struct perf_ftrace {
> >  	bool			trace_children;
> >  	unsigned		buffer_size_kb;
> >  	bool			no_pager;
> > +	unsigned		initial_delay;
> >  };
> >  
> >  struct filter_entry {
> > @@ -617,13 +618,23 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
> >  	/* display column headers */
> >  	read_tracing_file_to_stdout("trace");
> >  
> > -	if (write_tracing_file("tracing_on", "1") < 0) {
> > -		pr_err("can't enable tracing\n");
> > -		goto out_close_fd;
> > +	if (!ftrace->initial_delay) {
> > +		if (write_tracing_file("tracing_on", "1") < 0) {
> > +			pr_err("can't enable tracing\n");
> > +			goto out_close_fd;
> > +		}
> >  	}
> >  
> >  	perf_evlist__start_workload(ftrace->evlist);
> >  
> > +	if (ftrace->initial_delay) {
> > +		usleep(ftrace->initial_delay * 1000);
> > +		if (write_tracing_file("tracing_on", "1") < 0) {
> > +			pr_err("can't enable tracing\n");
> > +			goto out_close_fd;
> > +		}
> > +	}
> > +
> >  	while (!done) {
> >  		if (poll(&pollfd, 1, -1) < 0)
> >  			break;
> > @@ -747,6 +758,8 @@ int cmd_ftrace(int argc, const char **argv)
> >  		     "size of per cpu buffer in kb"),
> >  	OPT_BOOLEAN('P', "no-pager", &ftrace.no_pager,
> >  		    "Do not use pager"),
> > +	OPT_UINTEGER('d', "delay", &ftrace.initial_delay,
> > +		     "Wait <n> ms before tracing"),
> >  	OPT_END()
> >  	};
> >  
> > -- 
> > 2.25.1
> > 
> 
> -- 
> 
> - Arnaldo

-- 
Cheers,
Changbin Du

Re: [PATCH 16/19] perf ftrace: add option -t/--tid to filter by thread id

[Changbin Du]

On Wed, May 20, 2020 at 06:12:28PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Sun, May 10, 2020 at 11:06:25PM +0800, Changbin Du escreveu:
> > This allows us to trace single thread instead of the whole process.
> 
> I was going to adjust the patch to add, but you forgot to add the entry
> to tools/perf/Documentation/perf-ftrace.txt,
> 
> Please do so,
>
no problem. I will apend a new patch to update all the documentation.

> - Arnaldo
>  
> > Signed-off-by: Changbin Du <changbin.du@gmail.com>
> > ---
> >  tools/perf/builtin-ftrace.c | 2 ++
> >  1 file changed, 2 insertions(+)
> > 
> > diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> > index 0b39b6a88026..8d04e5afe2d3 100644
> > --- a/tools/perf/builtin-ftrace.c
> > +++ b/tools/perf/builtin-ftrace.c
> > @@ -707,6 +707,8 @@ int cmd_ftrace(int argc, const char **argv)
> >  		    "List available functions to filter"),
> >  	OPT_STRING('p', "pid", &ftrace.target.pid, "pid",
> >  		   "trace on existing process id"),
> > +	OPT_STRING('t', "tid", &ftrace.target.tid, "tid",
> > +		   "trace on existing thread id (exclusive to --pid)"),
> >  	OPT_INCR('v', "verbose", &verbose,
> >  		 "be more verbose"),
> >  	OPT_BOOLEAN('a', "all-cpus", &ftrace.target.system_wide,
> > -- 
> > 2.25.1
> > 
> 
> -- 
> 
> - Arnaldo

-- 
Cheers,
Changbin Du

Re: [PATCH 15/19] perf ftrace: show trace column header

[Changbin Du]

On Wed, May 20, 2020 at 06:11:01PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Sun, May 10, 2020 at 11:06:24PM +0800, Changbin Du escreveu:
> > This makes perf-ftrace display column header before printing trace.
> 
> [acme@five perf]$ perf report -h header
> 
>  Usage: perf report [<options>]
> 
>         --header          Show data header.
>         --header-only     Show only data header.
> 
> [acme@five perf]$
> 
> Perhaps there is value in --header-only to show the default that will be
> setup when using some set of options.
> 
Arnaldo, what is the '--header-only' option used for? As for ftrace, the header
format is different for different trace options.

> - Arnaldo
> 
> > $ sudo perf ftrace
> > \# tracer: function
> > \#
> > \# entries-in-buffer/entries-written: 0/0   #P:8
> > \#
> > \#           TASK-PID     CPU#   TIMESTAMP  FUNCTION
> > \#              | |         |       |         |
> >            <...>-9246  [006]  10726.262760: mutex_unlock <-rb_simple_write
> >            <...>-9246  [006]  10726.262764: __fsnotify_parent <-vfs_write
> >            <...>-9246  [006]  10726.262765: fsnotify <-vfs_write
> >            <...>-9246  [006]  10726.262766: __sb_end_write <-vfs_write
> >            <...>-9246  [006]  10726.262767: fpregs_assert_state_consistent <-do_syscall_64
> > 
> > Signed-off-by: Changbin Du <changbin.du@gmail.com>
> > ---
> >  tools/perf/builtin-ftrace.c | 3 +++
> >  1 file changed, 3 insertions(+)
> > 
> > diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> > index 64c22f367ba2..0b39b6a88026 100644
> > --- a/tools/perf/builtin-ftrace.c
> > +++ b/tools/perf/builtin-ftrace.c
> > @@ -614,6 +614,9 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
> >  	fcntl(trace_fd, F_SETFL, O_NONBLOCK);
> >  	pollfd.fd = trace_fd;
> >  
> > +	/* display column headers */
> > +	read_tracing_file_to_stdout("trace");
> > +
> >  	if (write_tracing_file("tracing_on", "1") < 0) {
> >  		pr_err("can't enable tracing\n");
> >  		goto out_close_fd;
> > -- 
> > 2.25.1
> > 
> 
> -- 
> 
> - Arnaldo

-- 
Cheers,
Changbin Du

Re: [PATCH 14/19] perf ftrace: add option -P/--no-pager to disable pager

[Changbin Du]

On Wed, May 20, 2020 at 06:09:14PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Sun, May 10, 2020 at 11:06:23PM +0800, Changbin Du escreveu:
> > Sometimes we want perf displays trace immediately. So this adds an option
> > '-P/--no-pager' to disable pager if needed.
> 
> 
> Try:
> 
> perf record sleep 5
> perf script
> 
> Then:
> 
> perf --no-pager script
> 
> I.e. its there already.
> 
> - Arnaldo
>
Yes, just tried this and it works:
perf --no-pager ftrace

So, the problem is that this option is not documented.
> > 
> > Signed-off-by: Changbin Du <changbin.du@gmail.com>
> > ---
> >  tools/perf/builtin-ftrace.c | 17 ++++++++++++++---
> >  1 file changed, 14 insertions(+), 3 deletions(-)
> > 
> > diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> > index a93fbdac6aa4..64c22f367ba2 100644
> > --- a/tools/perf/builtin-ftrace.c
> > +++ b/tools/perf/builtin-ftrace.c
> > @@ -48,6 +48,7 @@ struct perf_ftrace {
> >  	unsigned		tracing_thresh;
> >  	bool			trace_children;
> >  	unsigned		buffer_size_kb;
> > +	bool			no_pager;
> >  };
> >  
> >  struct filter_entry {
> > @@ -56,6 +57,7 @@ struct filter_entry {
> >  };
> >  
> >  static volatile int workload_exec_errno;
> > +static bool interrupted;
> >  static bool done;
> >  
> >  static void sig_handler(int sig __maybe_unused)
> > @@ -63,6 +65,12 @@ static void sig_handler(int sig __maybe_unused)
> >  	done = true;
> >  }
> >  
> > +static void sig_handler_int(int sig __maybe_unused)
> > +{
> > +	sig_handler(sig);
> > +	interrupted = 1;
> > +}
> > +
> >  /*
> >   * perf_evlist__prepare_workload will send a SIGUSR1 if the fork fails, since
> >   * we asked by setting its exec_error to the function below,
> > @@ -492,7 +500,7 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
> >  		return -1;
> >  	}
> >  
> > -	signal(SIGINT, sig_handler);
> > +	signal(SIGINT, sig_handler_int);
> >  	signal(SIGUSR1, sig_handler);
> >  	signal(SIGCHLD, sig_handler);
> >  	signal(SIGPIPE, sig_handler);
> > @@ -585,7 +593,8 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
> >  		goto out_reset;
> >  	}
> >  
> > -	setup_pager();
> > +	if (!ftrace->no_pager)
> > +		setup_pager();
> >  
> >  	trace_file = get_tracing_file("trace_pipe");
> >  	if (!trace_file) {
> > @@ -636,7 +645,7 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
> >  	}
> >  
> >  	/* read remaining buffer contents */
> > -	while (true) {
> > +	while (true && !interrupted) {
> >  		int n = read(trace_fd, buf, sizeof(buf));
> >  		if (n <= 0)
> >  			break;
> > @@ -731,6 +740,8 @@ int cmd_ftrace(int argc, const char **argv)
> >  		    "Trace children processes"),
> >  	OPT_UINTEGER('b', "buffer-size", &ftrace.buffer_size_kb,
> >  		     "size of per cpu buffer in kb"),
> > +	OPT_BOOLEAN('P', "no-pager", &ftrace.no_pager,
> > +		    "Do not use pager"),
> >  	OPT_END()
> >  	};
> >  
> > -- 
> > 2.25.1
> > 
> 
> -- 
> 
> - Arnaldo

-- 
Cheers,
Changbin Du

Re: [PATCH 05/19] perf ftrace: add option '-l/--list-functions' to list available functions

[Changbin Du]

On Wed, May 20, 2020 at 05:59:40PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Sun, May 10, 2020 at 11:06:14PM +0800, Changbin Du escreveu:
> > This adds an option '-l/--list-functions' to list all available functions
> > which is read from tracing file 'available_filter_functions'.
> 
> Here, in 'perf probe' we have:
> 
> [acme@five perf]$ perf probe -F tcp_* | head
> tcp_abort
> tcp_ack
> tcp_ack_update_rtt.isra.0
> tcp_add_backlog
> tcp_add_reno_sack
> tcp_adjust_pcount
> tcp_alloc_md5sig_pool
> tcp_any_retrans_done.part.0
> tcp_assign_congestion_control
> tcp_bpf_clone
> [acme@five perf]$ perf probe -h -F
> 
>  Usage: perf probe [<options>] 'PROBEDEF' ['PROBEDEF' ...]
>     or: perf probe [<options>] --add 'PROBEDEF' [--add 'PROBEDEF' ...]
>     or: perf probe [<options>] --del '[GROUP:]EVENT' ...
>     or: perf probe --list [GROUP:]EVENT ...
>     or: perf probe [<options>] --line 'LINEDESC'
>     or: perf probe [<options>] --vars 'PROBEPOINT'
>     or: perf probe [<options>] --funcs
> 
>     -F, --funcs <[FILTER]>
>                           Show potential probe-able functions.
> 
> [acme@five perf]$
> 
> So I think we should align with this.
> 
> - Arnaldo
>
Agreed. Will do that in next revision.

> > Signed-off-by: Changbin Du <changbin.du@gmail.com>
> > ---
> >  tools/perf/builtin-ftrace.c | 43 +++++++++++++++++++++++++++++++++++++
> >  1 file changed, 43 insertions(+)
> > 
> > diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> > index 1d30c2d5f88b..8133d910d5d8 100644
> > --- a/tools/perf/builtin-ftrace.c
> > +++ b/tools/perf/builtin-ftrace.c
> > @@ -33,6 +33,7 @@ struct perf_ftrace {
> >  	struct evlist		*evlist;
> >  	struct target		target;
> >  	const char		*tracer;
> > +	bool			list_avail_functions;
> >  	struct list_head	filters;
> >  	struct list_head	notrace;
> >  	struct list_head	graph_funcs;
> > @@ -142,6 +143,43 @@ static int write_tracing_option_file(const char *name, const char *val)
> >  	return ret;
> >  }
> >  
> > +static int read_tracing_file_to_stdout(const char *name)
> > +{
> > +	char buf[4096];
> > +	char *file;
> > +	int fd;
> > +	int ret = -1;
> > +
> > +	file = get_tracing_file(name);
> > +	if (!file) {
> > +		pr_debug("cannot get tracing file: %s\n", name);
> > +		return -1;
> > +	}
> > +
> > +	fd = open(file, O_RDONLY);
> > +	if (fd < 0) {
> > +		pr_debug("cannot open tracing file: %s: %s\n",
> > +			 name, str_error_r(errno, buf, sizeof(buf)));
> > +		goto out;
> > +	}
> > +
> > +	/* read contents to stdout */
> > +	while (true) {
> > +		int n = read(fd, buf, sizeof(buf));
> > +		if (n <= 0)
> > +			goto out_close;
> > +		if (fwrite(buf, n, 1, stdout) != 1)
> > +			goto out_close;
> > +	}
> > +	ret = 0;
> > +
> > +out_close:
> > +	close(fd);
> > +out:
> > +	put_tracing_file(file);
> > +	return ret;
> > +}
> > +
> >  static int reset_tracing_cpu(void);
> >  static void reset_tracing_filters(void);
> >  
> > @@ -332,6 +370,9 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
> >  	signal(SIGCHLD, sig_handler);
> >  	signal(SIGPIPE, sig_handler);
> >  
> > +	if (ftrace->list_avail_functions)
> > +		return read_tracing_file_to_stdout("available_filter_functions");
> > +
> >  	if (reset_tracing_files(ftrace) < 0) {
> >  		pr_err("failed to reset ftrace\n");
> >  		goto out;
> > @@ -483,6 +524,8 @@ int cmd_ftrace(int argc, const char **argv)
> >  		NULL
> >  	};
> >  	const struct option ftrace_options[] = {
> > +	OPT_BOOLEAN('L', "list-functions", &ftrace.list_avail_functions,
> > +		    "List available functions to filter"),
> >  	OPT_STRING('p', "pid", &ftrace.target.pid, "pid",
> >  		   "trace on existing process id"),
> >  	OPT_INCR('v', "verbose", &verbose,
> > -- 
> > 2.25.1
> > 
> 
> -- 
> 
> - Arnaldo

-- 
Cheers,
Changbin Du

Re: [PATCH 13/19] perf ftrace: add option '-b/--buffer-size' to set per-cpu buffer size

[Changbin Du]

On Wed, May 20, 2020 at 06:08:14PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Sun, May 10, 2020 at 11:06:22PM +0800, Changbin Du escreveu:
> > This adds an option '-b/--buffer-size' to allow us set the size of per-cpu
> > tracing buffer.
> 
> -m
>
sure.

> > Signed-off-by: Changbin Du <changbin.du@gmail.com>
> > ---
> >  tools/perf/builtin-ftrace.c | 45 ++++++++++++++++++++++++++++---------
> >  1 file changed, 35 insertions(+), 10 deletions(-)
> > 
> > diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> > index 8fd95c109fe8..a93fbdac6aa4 100644
> > --- a/tools/perf/builtin-ftrace.c
> > +++ b/tools/perf/builtin-ftrace.c
> > @@ -47,6 +47,7 @@ struct perf_ftrace {
> >  	bool			long_info;
> >  	unsigned		tracing_thresh;
> >  	bool			trace_children;
> > +	unsigned		buffer_size_kb;
> >  };
> >  
> >  struct filter_entry {
> > @@ -187,6 +188,17 @@ static int read_tracing_file_to_stdout(const char *name)
> >  	return ret;
> >  }
> >  
> > +static int write_tracing_file_int(const char *name, int value)
> > +{
> > +	char buf[16];
> > +
> > +	snprintf(buf, sizeof(buf), "%d", value);
> > +	if (write_tracing_file(name, buf) < 0)
> > +		return -1;
> > +
> > +	return 0;
> > +}
> > +
> >  static int reset_tracing_cpu(void);
> >  static void reset_tracing_filters(void);
> >  
> > @@ -360,8 +372,6 @@ static void reset_tracing_filters(void)
> >  
> >  static int set_tracing_depth(struct perf_ftrace *ftrace)
> >  {
> > -	char buf[16];
> > -
> >  	if (ftrace->graph_depth == 0)
> >  		return 0;
> >  
> > @@ -370,9 +380,7 @@ static int set_tracing_depth(struct perf_ftrace *ftrace)
> >  		return -1;
> >  	}
> >  
> > -	snprintf(buf, sizeof(buf), "%d", ftrace->graph_depth);
> > -
> > -	if (write_tracing_file("max_graph_depth", buf) < 0)
> > +	if (write_tracing_file_int("max_graph_depth", ftrace->graph_depth) < 0)
> >  		return -1;
> >  
> >  	return 0;
> > @@ -419,14 +427,10 @@ static int set_tracing_long_info(struct perf_ftrace *ftrace)
> >  
> >  static int set_tracing_thresh(struct perf_ftrace *ftrace)
> >  {
> > -	char buf[16];
> > -
> >  	if (ftrace->tracing_thresh == 0)
> >  		return 0;
> >  
> > -	snprintf(buf, sizeof(buf), "%d", ftrace->tracing_thresh);
> > -
> > -	if (write_tracing_file("tracing_thresh", buf) < 0)
> > +	if (write_tracing_file_int("tracing_thresh", ftrace->tracing_thresh) < 0)
> >  		return -1;
> >  
> >  	return 0;
> > @@ -454,6 +458,20 @@ static int set_tracing_trace_children(struct perf_ftrace *ftrace)
> >  	return 0;
> >  }
> >  
> > +static int set_tracing_buffer_size_kb(struct perf_ftrace *ftrace)
> > +{
> > +	int ret;
> > +
> > +	if (ftrace->buffer_size_kb == 0)
> > +		return 0;
> > +
> > +	ret = write_tracing_file_int("buffer_size_kb", ftrace->buffer_size_kb);
> > +	if (ret < 0)
> > +		return ret;
> > +
> > +	return 0;
> > +}
> > +
> >  static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
> >  {
> >  	char *trace_file;
> > @@ -557,6 +575,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
> >  		goto out_reset;
> >  	}
> >  
> > +	if (set_tracing_buffer_size_kb(ftrace) < 0) {
> > +		pr_err("failed to set tracing per-cpu buffer size\n");
> > +		goto out_reset;
> > +	}
> > +
> >  	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
> >  		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
> >  		goto out_reset;
> > @@ -706,6 +729,8 @@ int cmd_ftrace(int argc, const char **argv)
> >  		     "Only show functions of which the duration is greater than <n>µs"),
> >  	OPT_BOOLEAN(0, "trace-children", &ftrace.trace_children,
> >  		    "Trace children processes"),
> > +	OPT_UINTEGER('b', "buffer-size", &ftrace.buffer_size_kb,
> > +		     "size of per cpu buffer in kb"),
> >  	OPT_END()
> >  	};
> >  
> > -- 
> > 2.25.1
> > 
> 
> -- 
> 
> - Arnaldo

-- 
Cheers,
Changbin Du

Re: [PATCH 11/19] perf ftrace: add option '-u/--userstacktrace' to show userspace stacktrace

[Changbin Du]

On Wed, May 20, 2020 at 06:07:41PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Sun, May 10, 2020 at 11:06:20PM +0800, Changbin Du escreveu:
> > This adds an option ''-u/--userstacktrace' for function tracer to display
> > userspace back trace.
> 
> Probably we should have this as a term, an option to --call-graph?
> 
> For --call-graph the way to suppress this is to ask for the event to be
> in the kernel only, i.e. something like:
> 
> 	perf record -e cycles:k --call-graph
> 
> So perhaps we should have something like:
> 
> 	perf ftrace --call-graph 
> 
> With some default, possibly a bit different than the other perf tools,
> just including the kernel, and accepting:
> 
> 	perf ftrace --call-graph
> 	perf ftrace --call-graph k
> 	perf ftrace --call-graph u
> 	perf ftrace --call-graph uk
> 
> - Arnaldo
>
I just found userstacktrace is not supported by ftrace yet. It is not
hard to add this feature in kernel and I have done it locally. So I will
drop this option before it is upstreamed.

> > Signed-off-by: Changbin Du <changbin.du@gmail.com>
> > ---
> >  tools/perf/builtin-ftrace.c | 24 ++++++++++++++++++++++++
> >  1 file changed, 24 insertions(+)
> > 
> > diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> > index 2ef5d1c4b23c..ab76ba66bd9e 100644
> > --- a/tools/perf/builtin-ftrace.c
> > +++ b/tools/perf/builtin-ftrace.c
> > @@ -40,6 +40,7 @@ struct perf_ftrace {
> >  	struct list_head	nograph_funcs;
> >  	int			graph_depth;
> >  	bool			func_stack_trace;
> > +	bool			userstacktrace;
> >  	bool			nosleep_time;
> >  	bool			nofuncgraph_irqs;
> >  	bool			funcgraph_tail;
> > @@ -197,6 +198,8 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
> >  	write_tracing_option_file("funcgraph-proc", "0");
> >  	write_tracing_option_file("funcgraph-abstime", "0");
> >  	write_tracing_option_file("irq-info", "0");
> > +	write_tracing_option_file("userstacktrace", "0");
> > +	write_tracing_option_file("sym-userobj", "0");
> >  }
> >  
> >  static int reset_tracing_files(struct perf_ftrace *ftrace __maybe_unused)
> > @@ -287,6 +290,20 @@ static int set_tracing_func_stack_trace(struct perf_ftrace *ftrace)
> >  	return 0;
> >  }
> >  
> > +static int set_tracing_userstacktrace(struct perf_ftrace *ftrace)
> > +{
> > +	if (!ftrace->userstacktrace)
> > +		return 0;
> > +
> > +	if (write_tracing_option_file("userstacktrace", "1") < 0)
> > +		return -1;
> > +
> > +	if (write_tracing_option_file("sym-userobj", "1") < 0)
> > +		return -1;
> > +
> > +	return 0;
> > +}
> > +
> >  static int reset_tracing_cpu(void)
> >  {
> >  	struct perf_cpu_map *cpumap = perf_cpu_map__new(NULL);
> > @@ -482,6 +499,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
> >  		goto out_reset;
> >  	}
> >  
> > +	if (set_tracing_userstacktrace(ftrace) < 0) {
> > +		pr_err("failed to set tracing option userstacktrace\n");
> > +		goto out_reset;
> > +	}
> > +
> >  	if (set_tracing_filters(ftrace) < 0) {
> >  		pr_err("failed to set tracing filters\n");
> >  		goto out_reset;
> > @@ -644,6 +666,8 @@ int cmd_ftrace(int argc, const char **argv)
> >  		     "do not trace given functions", parse_filter_func),
> >  	OPT_BOOLEAN('s', "func-stack-trace", &ftrace.func_stack_trace,
> >  		    "Show kernel stack trace for function tracer"),
> > +	OPT_BOOLEAN('u', "userstacktrace", &ftrace.userstacktrace,
> > +		    "Show stacktrace of the current user space thread"),
> >  	OPT_CALLBACK_DEFAULT('G', "graph-funcs", &ftrace.graph_funcs, "func",
> >  		     "Set graph filter on given functions (imply to use function_graph tracer)",
> >  		     parse_filter_func, "*"),
> > -- 
> > 2.25.1
> > 
> 
> -- 
> 
> - Arnaldo

-- 
Cheers,
Changbin Du

Re: [PATCH 10/19] perf ftrace: add support for trace option funcgraph-tail

[Changbin Du]

Hi Arnaldo,
First appologize for so long dealy! Maybe you have forgot the context of this
serias ;)

On Wed, May 20, 2020 at 06:05:09PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Sun, May 10, 2020 at 11:06:19PM +0800, Changbin Du escreveu:
> > This adds an option '--funcgraph-tail' for function graph tracer.
> 
> And I think we should make these available in a compact way, as Intel PT
> has, i.e. instead of doing something like:
> 
>    --function-graph-options nosleep_time,noirqs,no_tail,long_info
> 
> We could have:
> 
>    -G ns,ni,nt,li
> 
> To save on typing, or something like that.
>
I just noticed that there is an existing graph tracer option:
	--graph-depth <n>

Then our new option would be:
	--graph-options depth=<n>,nosleep_time,noirqs,no_tail,long_info

This makes it a little complex to parse such options (need to parse key-vale
pairs).

So can we follow the existing style instead? I mean we can have below options:
	-D --graph-depth <n>
	-G --graph-funcs <func>

	--graph-nosleep-time
	--graph-noirqs
	--graph-notail
	--graph-longinfo

All graph tracer options are prefixed with '--graph-'.

> - Arnaldo
>  
> > Signed-off-by: Changbin Du <changbin.du@gmail.com>
> > ---
> >  tools/perf/builtin-ftrace.c | 20 ++++++++++++++++++++
> >  1 file changed, 20 insertions(+)
> > 
> > diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c
> > index 20bc14d6c5fb..2ef5d1c4b23c 100644
> > --- a/tools/perf/builtin-ftrace.c
> > +++ b/tools/perf/builtin-ftrace.c
> > @@ -42,6 +42,7 @@ struct perf_ftrace {
> >  	bool			func_stack_trace;
> >  	bool			nosleep_time;
> >  	bool			nofuncgraph_irqs;
> > +	bool			funcgraph_tail;
> >  	bool			long_info;
> >  	unsigned		tracing_thresh;
> >  };
> > @@ -192,6 +193,7 @@ static void reset_tracing_options(struct perf_ftrace *ftrace __maybe_unused)
> >  	write_tracing_option_file("func_stack_trace", "0");
> >  	write_tracing_option_file("sleep-time", "1");
> >  	write_tracing_option_file("funcgraph-irqs", "1");
> > +	write_tracing_option_file("funcgraph-tail", "0");
> >  	write_tracing_option_file("funcgraph-proc", "0");
> >  	write_tracing_option_file("funcgraph-abstime", "0");
> >  	write_tracing_option_file("irq-info", "0");
> > @@ -411,6 +413,17 @@ static int set_tracing_thresh(struct perf_ftrace *ftrace)
> >  	return 0;
> >  }
> >  
> > +static int set_tracing_funcgraph_tail(struct perf_ftrace *ftrace)
> > +{
> > +	if (!ftrace->funcgraph_tail)
> > +		return 0;
> > +
> > +	if (write_tracing_option_file("funcgraph-tail", "1") < 0)
> > +		return -1;
> > +
> > +	return 0;
> > +}
> > +
> >  static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
> >  {
> >  	char *trace_file;
> > @@ -499,6 +512,11 @@ static int __cmd_ftrace(struct perf_ftrace *ftrace, int argc, const char **argv)
> >  		goto out_reset;
> >  	}
> >  
> > +	if (set_tracing_funcgraph_tail(ftrace) < 0) {
> > +		pr_err("failed to set tracing option funcgraph-tail\n");
> > +		goto out_reset;
> > +	}
> > +
> >  	if (write_tracing_file("current_tracer", ftrace->tracer) < 0) {
> >  		pr_err("failed to set current_tracer to %s\n", ftrace->tracer);
> >  		goto out_reset;
> > @@ -638,6 +656,8 @@ int cmd_ftrace(int argc, const char **argv)
> >  		    "Measure on-CPU time only (function_graph only)"),
> >  	OPT_BOOLEAN(0, "nofuncgraph-irqs", &ftrace.nofuncgraph_irqs,
> >  		    "Ignore functions that happen inside interrupt (function_graph only)"),
> > +	OPT_BOOLEAN(0, "funcgraph-tail", &ftrace.funcgraph_tail,
> > +		    "Show function tails comment (function_graph only)"),
> >  	OPT_BOOLEAN('l', "long-info", &ftrace.long_info,
> >  		    "Show process names, PIDs, timestamps, irq-info if available"),
> >  	OPT_UINTEGER(0, "tracing-thresh", &ftrace.tracing_thresh,
> > -- 
> > 2.25.1
> > 
> 
> -- 
> 
> - Arnaldo

-- 
Cheers,
Changbin Du