All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH RFC V9 1/3] perf,tools: move callchain option parse code to util.c
@ 2015-08-06 19:44 kan.liang
  2015-08-06 19:44 ` [PATCH RFC V9 2/3] perf,tools: per-event callgraph support kan.liang
                   ` (3 more replies)
  0 siblings, 4 replies; 20+ messages in thread
From: kan.liang @ 2015-08-06 19:44 UTC (permalink / raw)
  To: acme, jolsa; +Cc: namhyung, ak, linux-kernel, Kan Liang

From: Kan Liang <kan.liang@intel.com>

Move callchain option parse related code to util.c

Signed-off-by: Kan Liang <kan.liang@intel.com>
---

Changes since V8:
 - Fix the link problems found by perf python test case.
   Introduce the patch to move callchain option parse related code to util.c
   
 tools/perf/util/callchain.c | 89 +-------------------------------------------
 tools/perf/util/callchain.h |  1 +
 tools/perf/util/util.c      | 90 +++++++++++++++++++++++++++++++++++++++++++++
 tools/perf/util/util.h      |  2 +
 4 files changed, 94 insertions(+), 88 deletions(-)

diff --git a/tools/perf/util/callchain.c b/tools/perf/util/callchain.c
index 931cca8..773fe13 100644
--- a/tools/perf/util/callchain.c
+++ b/tools/perf/util/callchain.c
@@ -25,96 +25,9 @@
 
 __thread struct callchain_cursor callchain_cursor;
 
-#ifdef HAVE_DWARF_UNWIND_SUPPORT
-static int get_stack_size(const char *str, unsigned long *_size)
-{
-	char *endptr;
-	unsigned long size;
-	unsigned long max_size = round_down(USHRT_MAX, sizeof(u64));
-
-	size = strtoul(str, &endptr, 0);
-
-	do {
-		if (*endptr)
-			break;
-
-		size = round_up(size, sizeof(u64));
-		if (!size || size > max_size)
-			break;
-
-		*_size = size;
-		return 0;
-
-	} while (0);
-
-	pr_err("callchain: Incorrect stack dump size (max %ld): %s\n",
-	       max_size, str);
-	return -1;
-}
-#endif /* HAVE_DWARF_UNWIND_SUPPORT */
-
 int parse_callchain_record_opt(const char *arg, struct callchain_param *param)
 {
-	char *tok, *name, *saveptr = NULL;
-	char *buf;
-	int ret = -1;
-
-	/* We need buffer that we know we can write to. */
-	buf = malloc(strlen(arg) + 1);
-	if (!buf)
-		return -ENOMEM;
-
-	strcpy(buf, arg);
-
-	tok = strtok_r((char *)buf, ",", &saveptr);
-	name = tok ? : (char *)buf;
-
-	do {
-		/* Framepointer style */
-		if (!strncmp(name, "fp", sizeof("fp"))) {
-			if (!strtok_r(NULL, ",", &saveptr)) {
-				param->record_mode = CALLCHAIN_FP;
-				ret = 0;
-			} else
-				pr_err("callchain: No more arguments "
-				       "needed for --call-graph fp\n");
-			break;
-
-#ifdef HAVE_DWARF_UNWIND_SUPPORT
-		/* Dwarf style */
-		} else if (!strncmp(name, "dwarf", sizeof("dwarf"))) {
-			const unsigned long default_stack_dump_size = 8192;
-
-			ret = 0;
-			param->record_mode = CALLCHAIN_DWARF;
-			param->dump_size = default_stack_dump_size;
-
-			tok = strtok_r(NULL, ",", &saveptr);
-			if (tok) {
-				unsigned long size = 0;
-
-				ret = get_stack_size(tok, &size);
-				param->dump_size = size;
-			}
-#endif /* HAVE_DWARF_UNWIND_SUPPORT */
-		} else if (!strncmp(name, "lbr", sizeof("lbr"))) {
-			if (!strtok_r(NULL, ",", &saveptr)) {
-				param->record_mode = CALLCHAIN_LBR;
-				ret = 0;
-			} else
-				pr_err("callchain: No more arguments "
-					"needed for --call-graph lbr\n");
-			break;
-		} else {
-			pr_err("callchain: Unknown --call-graph option "
-			       "value: %s\n", arg);
-			break;
-		}
-
-	} while (0);
-
-	free(buf);
-	return ret;
+	return parse_callchain_record(arg, param);
 }
 
 static int parse_callchain_mode(const char *value)
diff --git a/tools/perf/util/callchain.h b/tools/perf/util/callchain.h
index 68a32c2..acee2b3 100644
--- a/tools/perf/util/callchain.h
+++ b/tools/perf/util/callchain.h
@@ -177,6 +177,7 @@ int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *
 			bool hide_unresolved);
 
 extern const char record_callchain_help[];
+extern int parse_callchain_record(const char *arg, struct callchain_param *param);
 int parse_callchain_record_opt(const char *arg, struct callchain_param *param);
 int parse_callchain_report_opt(const char *arg);
 int perf_callchain_config(const char *var, const char *value);
diff --git a/tools/perf/util/util.c b/tools/perf/util/util.c
index edc2d63..f7adf12 100644
--- a/tools/perf/util/util.c
+++ b/tools/perf/util/util.c
@@ -566,6 +566,96 @@ unsigned long parse_tag_value(const char *str, struct parse_tag *tags)
 	return (unsigned long) -1;
 }
 
+int get_stack_size(const char *str, unsigned long *_size)
+{
+	char *endptr;
+	unsigned long size;
+	unsigned long max_size = round_down(USHRT_MAX, sizeof(u64));
+
+	size = strtoul(str, &endptr, 0);
+
+	do {
+		if (*endptr)
+			break;
+
+		size = round_up(size, sizeof(u64));
+		if (!size || size > max_size)
+			break;
+
+		*_size = size;
+		return 0;
+
+	} while (0);
+
+	pr_err("callchain: Incorrect stack dump size (max %ld): %s\n",
+	       max_size, str);
+	return -1;
+}
+
+int parse_callchain_record(const char *arg, struct callchain_param *param)
+{
+	char *tok, *name, *saveptr = NULL;
+	char *buf;
+	int ret = -1;
+
+	/* We need buffer that we know we can write to. */
+	buf = malloc(strlen(arg) + 1);
+	if (!buf)
+		return -ENOMEM;
+
+	strcpy(buf, arg);
+
+	tok = strtok_r((char *)buf, ",", &saveptr);
+	name = tok ? : (char *)buf;
+
+	do {
+		/* Framepointer style */
+		if (!strncmp(name, "fp", sizeof("fp"))) {
+			if (!strtok_r(NULL, ",", &saveptr)) {
+				param->record_mode = CALLCHAIN_FP;
+				ret = 0;
+			} else
+				pr_err("callchain: No more arguments "
+				       "needed for --call-graph fp\n");
+			break;
+
+#ifdef HAVE_DWARF_UNWIND_SUPPORT
+		/* Dwarf style */
+		} else if (!strncmp(name, "dwarf", sizeof("dwarf"))) {
+			const unsigned long default_stack_dump_size = 8192;
+
+			ret = 0;
+			param->record_mode = CALLCHAIN_DWARF;
+			param->dump_size = default_stack_dump_size;
+
+			tok = strtok_r(NULL, ",", &saveptr);
+			if (tok) {
+				unsigned long size = 0;
+
+				ret = get_stack_size(tok, &size);
+				param->dump_size = size;
+			}
+#endif /* HAVE_DWARF_UNWIND_SUPPORT */
+		} else if (!strncmp(name, "lbr", sizeof("lbr"))) {
+			if (!strtok_r(NULL, ",", &saveptr)) {
+				param->record_mode = CALLCHAIN_LBR;
+				ret = 0;
+			} else
+				pr_err("callchain: No more arguments "
+					"needed for --call-graph lbr\n");
+			break;
+		} else {
+			pr_err("callchain: Unknown --call-graph option "
+			       "value: %s\n", arg);
+			break;
+		}
+
+	} while (0);
+
+	free(buf);
+	return ret;
+}
+
 int filename__read_str(const char *filename, char **buf, size_t *sizep)
 {
 	size_t size = 0, alloc_size = 0;
diff --git a/tools/perf/util/util.h b/tools/perf/util/util.h
index 20d625a..8148703 100644
--- a/tools/perf/util/util.h
+++ b/tools/perf/util/util.h
@@ -351,4 +351,6 @@ static inline char *asprintf_expr_not_in_ints(const char *var, size_t nints, int
 	return asprintf_expr_inout_ints(var, false, nints, ints);
 }
 
+int get_stack_size(const char *str, unsigned long *_size);
+
 #endif /* GIT_COMPAT_UTIL_H */
-- 
1.8.3.1


^ permalink raw reply related	[flat|nested] 20+ messages in thread

* [PATCH RFC V9 2/3] perf,tools: per-event callgraph support
  2015-08-06 19:44 [PATCH RFC V9 1/3] perf,tools: move callchain option parse code to util.c kan.liang
@ 2015-08-06 19:44 ` kan.liang
  2015-08-07 10:54   ` Jiri Olsa
  2015-08-07 15:38   ` Arnaldo Carvalho de Melo
  2015-08-06 19:44 ` [PATCH RFC V9 3/3] perf,tests: Add tests to callgraph and time parse kan.liang
                   ` (2 subsequent siblings)
  3 siblings, 2 replies; 20+ messages in thread
From: kan.liang @ 2015-08-06 19:44 UTC (permalink / raw)
  To: acme, jolsa; +Cc: namhyung, ak, linux-kernel, Kan Liang

From: Kan Liang <kan.liang@intel.com>

When multiple events are sampled it may not be needed to collect
callgraphs for all of them. The sample sites are usually nearby, and
it's enough to collect the callgraphs on a reference event (such as
precise cycles or precise instructions).
This patchkit adds the ability to turn off callgraphs and time stamp
per event. This in term can reduce sampling overhead and the size of the
perf.data. Furthermore, it makes collecting back traces and timestamps
possible when PEBS threshold > 1, which significantly reducing the
sampling overhead especially for frequently occurring events
(https://lkml.org/lkml/2015/5/10/196). For example, A slower event with
a larger period collects back traces/timestamps. Other more events run
fast with multi-pebs. The time stamps from the slower events can be used
to order the faster events. Their backtraces can give the user enough
hint to find the right spot.

Here are some examples and test results.

1. Comparing the elapsed time and perf.data size from "kernbench -M -H".

 The test command for FULL callgraph and time support.
   "perf record -e
   '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}'
   --call-graph fp --time"

 The test command for PARTIAL callgraph and time support.
   "perf record -e
   '{cpu/cpu-cycles,call-graph=fp,time,period=100000/,
     cpu/instructions,call-graph=no,time=0,period=20000/p}'"

 The elapsed time for FULL is 24.3 Sec, while for PARTIAL is 16.9 Sec.
 The perf.data size for FULL is 22.1 Gb, while for PARTIAL is 12.4 Gb.

2. Comparing the perf.data size and callgraph results.

 The test command for FULL callgraph and time support.
   "perf record -e
   '{cpu/cpu-cycles,period=100000/pp,cpu/instructions,period=20000/p}'
   --call-graph fp -- ./tchain_edit"

 The test command for PARTIAL callgraph and time support.
   "perf record -e
   '{cpu/cpu-cycles,call-graph=fp,time,period=100000/pp,
     cpu/instructions,call-graph=no,time=0,period=20000/p}'
   -- ./tchain_edit"

 The perf.data size for FULL is 43.2 MB, while for PARTIAL is 21.1 MB.
 The callgraph is roughly the same.

 The callgraph from FULL
 # Samples: 87K of event
 'cpu/cpu-cycles,call-graph=fp,time,period=100000/pp'
 # Event count (approx.): 8760000000
 #
 # Children      Self  Command      Shared Object       Symbol
 # ........  ........  ...........  ..................
..........................................
 #
    99.98%     0.00%  tchain_edit  libc-2.15.so        [.]
__libc_start_main
            |
            ---__libc_start_main

    99.97%     0.00%  tchain_edit  tchain_edit         [.] main
            |
            ---main
               __libc_start_main

    99.97%     0.00%  tchain_edit  tchain_edit         [.] f1
            |
            ---f1
               main
               __libc_start_main

    99.85%    87.01%  tchain_edit  tchain_edit         [.] f3
            |
            ---f3
               |
               |--99.74%-- f2
               |          f1
               |          main
               |          __libc_start_main
                --0.26%-- [...]
    99.71%     0.12%  tchain_edit  tchain_edit         [.] f2
            |
            ---f2
               f1
               main
               __libc_start_main

 The callgraph from PARTIAL
 # Samples: 417K of event
 'cpu/instructions,call-graph=no,time=0,period=20000/p'
 # Event count (approx.): 8346980000
 #
 # Children      Self  Command      Shared Object     Symbol
 # ........  ........  ...........  ................
..........................................
 #
    98.82%     0.00%  tchain_edit  libc-2.15.so      [.]
__libc_start_main
            |
            ---__libc_start_main

    98.82%     0.00%  tchain_edit  tchain_edit       [.] main
            |
            ---main
               __libc_start_main

    98.82%     0.00%  tchain_edit  tchain_edit       [.] f1
            |
            ---f1
               main
               __libc_start_main

    98.82%    98.28%  tchain_edit  tchain_edit       [.] f3
            |
            ---f3
               |
               |--0.53%-- f2
               |          f1
               |          main
               |          __libc_start_main
               |
               |--0.01%-- f1
               |          main
               |          __libc_start_main
                --99.46%-- [...]
    97.63%     0.03%  tchain_edit  tchain_edit       [.] f2
            |
            ---f2
               f1
               main
               __libc_start_main

     7.13%     0.03%  tchain_edit  [kernel.vmlinux]  [k] do_nmi
            |
            ---do_nmi
               end_repeat_nmi
               f3
               f2
               f1
               main
               __libc_start_main

Signed-off-by: Kan Liang <kan.liang@intel.com>
---

Changes since V8:
 - Same as global dump_size caculation, round_up dump_size for per-event dump_size.

 tools/perf/Documentation/perf-record.txt |  4 ++
 tools/perf/util/evsel.c                  | 67 +++++++++++++++++++++++++++++++-
 tools/perf/util/evsel.h                  |  4 ++
 tools/perf/util/parse-events.c           | 12 ++++++
 tools/perf/util/parse-events.h           |  2 +
 tools/perf/util/parse-events.l           |  2 +
 tools/perf/util/pmu.c                    |  3 +-
 7 files changed, 91 insertions(+), 3 deletions(-)

diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt
index 0d852d1..e633711 100644
--- a/tools/perf/Documentation/perf-record.txt
+++ b/tools/perf/Documentation/perf-record.txt
@@ -52,6 +52,10 @@ OPTIONS
 	  - 'time': Disable/enable time stamping. Acceptable values are 1 for
 		    enabling time stamping. 0 for disabling time stamping.
 		    The default is 1.
+	  - 'call-graph': Disable/enable callgraph. Acceptable str are "fp" for
+			 FP mode, "dwarf" for DWARF mode, "lbr" for LBR mode and
+			 "no" for disable callgraph.
+	  - 'stack-size': user stack size for dwarf mode
 	  Note: If user explicitly sets options which conflict with the params,
 	  the value set by the params will be overridden.
 
diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c
index f572f46..d902cad 100644
--- a/tools/perf/util/evsel.c
+++ b/tools/perf/util/evsel.c
@@ -588,11 +588,36 @@ perf_evsel__config_callgraph(struct perf_evsel *evsel,
 	}
 }
 
-static void apply_config_terms(struct perf_evsel *evsel)
+static void
+perf_evsel__reset_callgraph(struct perf_evsel *evsel,
+			    struct callchain_param *param)
+{
+	struct perf_event_attr *attr = &evsel->attr;
+
+	perf_evsel__reset_sample_bit(evsel, CALLCHAIN);
+	if (param->record_mode == CALLCHAIN_LBR) {
+		perf_evsel__reset_sample_bit(evsel, BRANCH_STACK);
+		attr->branch_sample_type &= ~(PERF_SAMPLE_BRANCH_USER |
+					      PERF_SAMPLE_BRANCH_CALL_STACK);
+	}
+	if (param->record_mode == CALLCHAIN_DWARF) {
+		perf_evsel__reset_sample_bit(evsel, REGS_USER);
+		perf_evsel__reset_sample_bit(evsel, STACK_USER);
+	}
+}
+
+static void apply_config_terms(struct perf_evsel *evsel,
+			       struct record_opts *opts)
 {
 	struct perf_evsel_config_term *term;
 	struct list_head *config_terms = &evsel->config_terms;
 	struct perf_event_attr *attr = &evsel->attr;
+	struct callchain_param param;
+	u32 dump_size = 0;
+	char *callgraph_buf = NULL;
+
+	/* callgraph default */
+	param.record_mode = callchain_param.record_mode;
 
 	list_for_each_entry(term, config_terms, list) {
 		switch (term->type) {
@@ -605,10 +630,48 @@ static void apply_config_terms(struct perf_evsel *evsel)
 			else
 				perf_evsel__reset_sample_bit(evsel, TIME);
 			break;
+		case PERF_EVSEL__CONFIG_TERM_CALLGRAPH:
+			callgraph_buf = term->val.callgraph;
+			break;
+		case PERF_EVSEL__CONFIG_TERM_STACK_USER:
+			dump_size = term->val.stack_user;
+			break;
 		default:
 			break;
 		}
 	}
+
+	/* User explicitly set per-event callgraph, clear the old setting and reset. */
+	if ((callgraph_buf != NULL) || (dump_size > 0)) {
+
+		/* parse callgraph parameters */
+		if (callgraph_buf != NULL) {
+			if (!strcmp(callgraph_buf, "no")) {
+				param.enabled = false;
+				param.record_mode = CALLCHAIN_NONE;
+			} else {
+				param.enabled = true;
+				if (parse_callchain_record(callgraph_buf, &param)) {
+					pr_err("per-event callgraph setting for %s failed. "
+					       "Apply callgraph global setting for it\n",
+					       evsel->name);
+					return;
+				}
+			}
+		}
+		if (dump_size > 0) {
+			dump_size = round_up(dump_size, sizeof(u64));
+			param.dump_size = dump_size;
+		}
+
+		/* If global callgraph set, clear it */
+		if (callchain_param.enabled)
+			perf_evsel__reset_callgraph(evsel, &callchain_param);
+
+		/* set perf-event callgraph */
+		if (param.enabled)
+			perf_evsel__config_callgraph(evsel, opts, &param);
+	}
 }
 
 /*
@@ -807,7 +870,7 @@ void perf_evsel__config(struct perf_evsel *evsel, struct record_opts *opts)
 	 * Apply event specific term settings,
 	 * it overloads any global configuration.
 	 */
-	apply_config_terms(evsel);
+	apply_config_terms(evsel, opts);
 }
 
 static int perf_evsel__alloc_fd(struct perf_evsel *evsel, int ncpus, int nthreads)
diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h
index 6a12908..09a3022 100644
--- a/tools/perf/util/evsel.h
+++ b/tools/perf/util/evsel.h
@@ -40,6 +40,8 @@ struct cgroup_sel;
 enum {
 	PERF_EVSEL__CONFIG_TERM_PERIOD,
 	PERF_EVSEL__CONFIG_TERM_TIME,
+	PERF_EVSEL__CONFIG_TERM_CALLGRAPH,
+	PERF_EVSEL__CONFIG_TERM_STACK_USER,
 	PERF_EVSEL__CONFIG_TERM_MAX,
 };
 
@@ -49,6 +51,8 @@ struct perf_evsel_config_term {
 	union {
 		u64	period;
 		bool	time;
+		char	*callgraph;
+		u64	stack_user;
 	} val;
 };
 
diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c
index 828936d..773fe8b 100644
--- a/tools/perf/util/parse-events.c
+++ b/tools/perf/util/parse-events.c
@@ -611,6 +611,12 @@ do {									   \
 			return -EINVAL;
 		}
 		break;
+	case PARSE_EVENTS__TERM_TYPE_CALLGRAPH:
+		CHECK_TYPE_VAL(STR);
+		break;
+	case PARSE_EVENTS__TERM_TYPE_STACKSIZE:
+		CHECK_TYPE_VAL(NUM);
+		break;
 	case PARSE_EVENTS__TERM_TYPE_NAME:
 		CHECK_TYPE_VAL(STR);
 		break;
@@ -662,6 +668,12 @@ do {								\
 		case PARSE_EVENTS__TERM_TYPE_TIME:
 			ADD_CONFIG_TERM(TIME, time, term->val.num);
 			break;
+		case PARSE_EVENTS__TERM_TYPE_CALLGRAPH:
+			ADD_CONFIG_TERM(CALLGRAPH, callgraph, term->val.str);
+			break;
+		case PARSE_EVENTS__TERM_TYPE_STACKSIZE:
+			ADD_CONFIG_TERM(STACK_USER, stack_user, term->val.num);
+			break;
 		default:
 			break;
 		}
diff --git a/tools/perf/util/parse-events.h b/tools/perf/util/parse-events.h
index e6f9aacc..87dc9f6 100644
--- a/tools/perf/util/parse-events.h
+++ b/tools/perf/util/parse-events.h
@@ -64,6 +64,8 @@ enum {
 	PARSE_EVENTS__TERM_TYPE_SAMPLE_PERIOD,
 	PARSE_EVENTS__TERM_TYPE_BRANCH_SAMPLE_TYPE,
 	PARSE_EVENTS__TERM_TYPE_TIME,
+	PARSE_EVENTS__TERM_TYPE_CALLGRAPH,
+	PARSE_EVENTS__TERM_TYPE_STACKSIZE,
 };
 
 struct parse_events_term {
diff --git a/tools/perf/util/parse-events.l b/tools/perf/util/parse-events.l
index f542750..1665497 100644
--- a/tools/perf/util/parse-events.l
+++ b/tools/perf/util/parse-events.l
@@ -184,6 +184,8 @@ name			{ return term(yyscanner, PARSE_EVENTS__TERM_TYPE_NAME); }
 period			{ return term(yyscanner, PARSE_EVENTS__TERM_TYPE_SAMPLE_PERIOD); }
 branch_type		{ return term(yyscanner, PARSE_EVENTS__TERM_TYPE_BRANCH_SAMPLE_TYPE); }
 time			{ return term(yyscanner, PARSE_EVENTS__TERM_TYPE_TIME); }
+call-graph		{ return term(yyscanner, PARSE_EVENTS__TERM_TYPE_CALLGRAPH); }
+stack-size		{ return term(yyscanner, PARSE_EVENTS__TERM_TYPE_STACKSIZE); }
 ,			{ return ','; }
 "/"			{ BEGIN(INITIAL); return '/'; }
 {name_minus}		{ return str(yyscanner, PE_NAME); }
diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c
index d4b0e64..239a653 100644
--- a/tools/perf/util/pmu.c
+++ b/tools/perf/util/pmu.c
@@ -634,7 +634,8 @@ static char *formats_error_string(struct list_head *formats)
 {
 	struct perf_pmu_format *format;
 	char *err, *str;
-	static const char *static_terms = "config,config1,config2,name,period,branch_type,time\n";
+	static const char *static_terms = "config,config1,config2,name,period,"
+					  "branch_type,time,call-graph,stack-size\n";
 	unsigned i = 0;
 
 	if (!asprintf(&str, "valid terms:"))
-- 
1.8.3.1


^ permalink raw reply related	[flat|nested] 20+ messages in thread

* [PATCH RFC V9 3/3] perf,tests: Add tests to callgraph and time parse
  2015-08-06 19:44 [PATCH RFC V9 1/3] perf,tools: move callchain option parse code to util.c kan.liang
  2015-08-06 19:44 ` [PATCH RFC V9 2/3] perf,tools: per-event callgraph support kan.liang
@ 2015-08-06 19:44 ` kan.liang
  2015-08-07 10:54   ` Jiri Olsa
  2015-08-07 10:51 ` [PATCH] perf tools: Move perf_counts struct and functions into separate object Jiri Olsa
  2015-08-12 12:29 ` [tip:perf/core] perf callchain: Move option parsing code to util.c tip-bot for Kan Liang
  3 siblings, 1 reply; 20+ messages in thread
From: kan.liang @ 2015-08-06 19:44 UTC (permalink / raw)
  To: acme, jolsa; +Cc: namhyung, ak, linux-kernel, Kan Liang

From: Kan Liang <kan.liang@intel.com>

Add tests in tests/parse-events.c to check call-graph and time option

Signed-off-by: Kan Liang <kan.liang@intel.com>
---
 tools/perf/tests/parse-events.c | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/tools/perf/tests/parse-events.c b/tools/perf/tests/parse-events.c
index f65bb89..9b6b2b63 100644
--- a/tools/perf/tests/parse-events.c
+++ b/tools/perf/tests/parse-events.c
@@ -479,6 +479,39 @@ static int test__checkevent_pmu_name(struct perf_evlist *evlist)
 	return 0;
 }
 
+static int test__checkevent_pmu_partial_time_callgraph(struct perf_evlist *evlist)
+{
+	struct perf_evsel *evsel = perf_evlist__first(evlist);
+
+	/* cpu/config=1,call-graph=fp,time,period=100000/ */
+	TEST_ASSERT_VAL("wrong number of entries", 2 == evlist->nr_entries);
+	TEST_ASSERT_VAL("wrong type", PERF_TYPE_RAW == evsel->attr.type);
+	TEST_ASSERT_VAL("wrong config",  1 == evsel->attr.config);
+	/*
+	 * The period, time and callgraph value gets configured
+	 * within perf_evlist__config,
+	 * while this test executes only parse events method.
+	 */
+	TEST_ASSERT_VAL("wrong period",     0 == evsel->attr.sample_period);
+	TEST_ASSERT_VAL("wrong callgraph",  !(PERF_SAMPLE_CALLCHAIN & evsel->attr.sample_type));
+	TEST_ASSERT_VAL("wrong time",  !(PERF_SAMPLE_TIME & evsel->attr.sample_type));
+
+	/* cpu/config=2,call-graph=no,time=0,period=2000/ */
+	evsel = perf_evsel__next(evsel);
+	TEST_ASSERT_VAL("wrong type", PERF_TYPE_RAW == evsel->attr.type);
+	TEST_ASSERT_VAL("wrong config",  2 == evsel->attr.config);
+	/*
+	 * The period, time and callgraph value gets configured
+	 * within perf_evlist__config,
+	 * while this test executes only parse events method.
+	 */
+	TEST_ASSERT_VAL("wrong period",     0 == evsel->attr.sample_period);
+	TEST_ASSERT_VAL("wrong callgraph",  !(PERF_SAMPLE_CALLCHAIN & evsel->attr.sample_type));
+	TEST_ASSERT_VAL("wrong time",  !(PERF_SAMPLE_TIME & evsel->attr.sample_type));
+
+	return 0;
+}
+
 static int test__checkevent_pmu_events(struct perf_evlist *evlist)
 {
 	struct perf_evsel *evsel = perf_evlist__first(evlist);
@@ -1555,6 +1588,11 @@ static struct evlist_test test__events_pmu[] = {
 		.check = test__checkevent_pmu_name,
 		.id    = 1,
 	},
+	{
+		.name  = "cpu/config=1,call-graph=fp,time,period=100000/,cpu/config=2,call-graph=no,time=0,period=2000/",
+		.check = test__checkevent_pmu_partial_time_callgraph,
+		.id    = 2,
+	},
 };
 
 struct terms_test {
-- 
1.8.3.1


^ permalink raw reply related	[flat|nested] 20+ messages in thread

* [PATCH] perf tools: Move perf_counts struct and functions into separate object
  2015-08-06 19:44 [PATCH RFC V9 1/3] perf,tools: move callchain option parse code to util.c kan.liang
  2015-08-06 19:44 ` [PATCH RFC V9 2/3] perf,tools: per-event callgraph support kan.liang
  2015-08-06 19:44 ` [PATCH RFC V9 3/3] perf,tests: Add tests to callgraph and time parse kan.liang
@ 2015-08-07 10:51 ` Jiri Olsa
  2015-08-07 13:14   ` Arnaldo Carvalho de Melo
  2015-08-12 12:28   ` [tip:perf/core] perf stat: " tip-bot for Jiri Olsa
  2015-08-12 12:29 ` [tip:perf/core] perf callchain: Move option parsing code to util.c tip-bot for Kan Liang
  3 siblings, 2 replies; 20+ messages in thread
From: Jiri Olsa @ 2015-08-07 10:51 UTC (permalink / raw)
  To: kan.liang; +Cc: acme, jolsa, namhyung, ak, linux-kernel

On Thu, Aug 06, 2015 at 03:44:52PM -0400, kan.liang@intel.com wrote:
> From: Kan Liang <kan.liang@intel.com>
> 
> Move callchain option parse related code to util.c

little nore about the reason would be nice ;-)

looks ok, but the python test is still failing,
the reason is the perf_counts struct objects and functions
bindings to util/stat.c object

Arnaldo,
I separated the 'struct perf_counts' into util/counts.c,
because I recall you did not want them in evsel.c,
please check attached patch.. it makes the python test
pass again (on top of Kan's change).

thanks,
jirka


---
Moving perf_counts struct and functions into separate object,
so we could remove stat.c object dependency from python build.

It makes the python code to be built properly, because now it
fails to load due to missing stat-shadow.c object dependency.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
---
diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c
index a054ddc0b2a0..7aa039bd379a 100644
--- a/tools/perf/builtin-stat.c
+++ b/tools/perf/builtin-stat.c
@@ -58,6 +58,7 @@
 #include "util/cpumap.h"
 #include "util/thread.h"
 #include "util/thread_map.h"
+#include "util/counts.h"
 
 #include <stdlib.h>
 #include <sys/prctl.h>
diff --git a/tools/perf/util/Build b/tools/perf/util/Build
index a1e5168dc1fb..4ca481cd38e6 100644
--- a/tools/perf/util/Build
+++ b/tools/perf/util/Build
@@ -67,6 +67,7 @@ libperf-y += target.o
 libperf-y += rblist.o
 libperf-y += intlist.o
 libperf-y += vdso.o
+libperf-y += counts.o
 libperf-y += stat.o
 libperf-y += stat-shadow.o
 libperf-y += record.o
diff --git a/tools/perf/util/counts.c b/tools/perf/util/counts.c
new file mode 100644
index 000000000000..e3fde313deb2
--- /dev/null
+++ b/tools/perf/util/counts.c
@@ -0,0 +1,52 @@
+#include <stdlib.h>
+#include "evsel.h"
+#include "counts.h"
+
+struct perf_counts *perf_counts__new(int ncpus, int nthreads)
+{
+	struct perf_counts *counts = zalloc(sizeof(*counts));
+
+	if (counts) {
+		struct xyarray *values;
+
+		values = xyarray__new(ncpus, nthreads, sizeof(struct perf_counts_values));
+		if (!values) {
+			free(counts);
+			return NULL;
+		}
+
+		counts->values = values;
+	}
+
+	return counts;
+}
+
+void perf_counts__delete(struct perf_counts *counts)
+{
+	if (counts) {
+		xyarray__delete(counts->values);
+		free(counts);
+	}
+}
+
+static void perf_counts__reset(struct perf_counts *counts)
+{
+	xyarray__reset(counts->values);
+}
+
+void perf_evsel__reset_counts(struct perf_evsel *evsel)
+{
+	perf_counts__reset(evsel->counts);
+}
+
+int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads)
+{
+	evsel->counts = perf_counts__new(ncpus, nthreads);
+	return evsel->counts != NULL ? 0 : -ENOMEM;
+}
+
+void perf_evsel__free_counts(struct perf_evsel *evsel)
+{
+	perf_counts__delete(evsel->counts);
+	evsel->counts = NULL;
+}
diff --git a/tools/perf/util/counts.h b/tools/perf/util/counts.h
new file mode 100644
index 000000000000..34d8baaf558a
--- /dev/null
+++ b/tools/perf/util/counts.h
@@ -0,0 +1,37 @@
+#ifndef __PERF_COUNTS_H
+#define __PERF_COUNTS_H
+
+#include "xyarray.h"
+
+struct perf_counts_values {
+	union {
+		struct {
+			u64 val;
+			u64 ena;
+			u64 run;
+		};
+		u64 values[3];
+	};
+};
+
+struct perf_counts {
+	s8			  scaled;
+	struct perf_counts_values aggr;
+	struct xyarray		  *values;
+};
+
+
+static inline struct perf_counts_values*
+perf_counts(struct perf_counts *counts, int cpu, int thread)
+{
+	return xyarray__entry(counts->values, cpu, thread);
+}
+
+struct perf_counts *perf_counts__new(int ncpus, int nthreads);
+void perf_counts__delete(struct perf_counts *counts);
+
+void perf_evsel__reset_counts(struct perf_evsel *evsel);
+int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads);
+void perf_evsel__free_counts(struct perf_evsel *evsel);
+
+#endif /* __PERF_COUNTS_H */
diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h
index 09a3022fa2c6..0ca0002db99a 100644
--- a/tools/perf/util/evsel.h
+++ b/tools/perf/util/evsel.h
@@ -9,7 +9,7 @@
 #include "xyarray.h"
 #include "symbol.h"
 #include "cpumap.h"
-#include "stat.h"
+#include "counts.h"
 
 struct perf_evsel;
 
diff --git a/tools/perf/util/python-ext-sources b/tools/perf/util/python-ext-sources
index 0766d98c5da5..51be28b1bca2 100644
--- a/tools/perf/util/python-ext-sources
+++ b/tools/perf/util/python-ext-sources
@@ -16,7 +16,7 @@ util/util.c
 util/xyarray.c
 util/cgroup.c
 util/rblist.c
-util/stat.c
+util/counts.c
 util/strlist.c
 util/trace-event.c
 ../lib/rbtree.c
diff --git a/tools/perf/util/stat.c b/tools/perf/util/stat.c
index c5c709cdc3ce..415c359de465 100644
--- a/tools/perf/util/stat.c
+++ b/tools/perf/util/stat.c
@@ -97,55 +97,6 @@ void perf_stat_evsel_id_init(struct perf_evsel *evsel)
 	}
 }
 
-struct perf_counts *perf_counts__new(int ncpus, int nthreads)
-{
-	struct perf_counts *counts = zalloc(sizeof(*counts));
-
-	if (counts) {
-		struct xyarray *values;
-
-		values = xyarray__new(ncpus, nthreads, sizeof(struct perf_counts_values));
-		if (!values) {
-			free(counts);
-			return NULL;
-		}
-
-		counts->values = values;
-	}
-
-	return counts;
-}
-
-void perf_counts__delete(struct perf_counts *counts)
-{
-	if (counts) {
-		xyarray__delete(counts->values);
-		free(counts);
-	}
-}
-
-static void perf_counts__reset(struct perf_counts *counts)
-{
-	xyarray__reset(counts->values);
-}
-
-void perf_evsel__reset_counts(struct perf_evsel *evsel)
-{
-	perf_counts__reset(evsel->counts);
-}
-
-int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads)
-{
-	evsel->counts = perf_counts__new(ncpus, nthreads);
-	return evsel->counts != NULL ? 0 : -ENOMEM;
-}
-
-void perf_evsel__free_counts(struct perf_evsel *evsel)
-{
-	perf_counts__delete(evsel->counts);
-	evsel->counts = NULL;
-}
-
 void perf_evsel__reset_stat_priv(struct perf_evsel *evsel)
 {
 	int i;
diff --git a/tools/perf/util/stat.h b/tools/perf/util/stat.h
index 0b897b083682..62448c8175d3 100644
--- a/tools/perf/util/stat.h
+++ b/tools/perf/util/stat.h
@@ -33,23 +33,6 @@ enum aggr_mode {
 	AGGR_THREAD,
 };
 
-struct perf_counts_values {
-	union {
-		struct {
-			u64 val;
-			u64 ena;
-			u64 run;
-		};
-		u64 values[3];
-	};
-};
-
-struct perf_counts {
-	s8			  scaled;
-	struct perf_counts_values aggr;
-	struct xyarray		  *values;
-};
-
 struct perf_stat_config {
 	enum aggr_mode	aggr_mode;
 	bool		scale;
@@ -57,12 +40,6 @@ struct perf_stat_config {
 	unsigned int	interval;
 };
 
-static inline struct perf_counts_values*
-perf_counts(struct perf_counts *counts, int cpu, int thread)
-{
-	return xyarray__entry(counts->values, cpu, thread);
-}
-
 void update_stats(struct stats *stats, u64 val);
 double avg_stats(struct stats *stats);
 double stddev_stats(struct stats *stats);
@@ -96,13 +73,6 @@ void perf_stat__update_shadow_stats(struct perf_evsel *counter, u64 *count,
 void perf_stat__print_shadow_stats(FILE *out, struct perf_evsel *evsel,
 				   double avg, int cpu, enum aggr_mode aggr);
 
-struct perf_counts *perf_counts__new(int ncpus, int nthreads);
-void perf_counts__delete(struct perf_counts *counts);
-
-void perf_evsel__reset_counts(struct perf_evsel *evsel);
-int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads);
-void perf_evsel__free_counts(struct perf_evsel *evsel);
-
 void perf_evsel__reset_stat_priv(struct perf_evsel *evsel);
 int perf_evsel__alloc_stat_priv(struct perf_evsel *evsel);
 void perf_evsel__free_stat_priv(struct perf_evsel *evsel);

^ permalink raw reply related	[flat|nested] 20+ messages in thread

* Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support
  2015-08-06 19:44 ` [PATCH RFC V9 2/3] perf,tools: per-event callgraph support kan.liang
@ 2015-08-07 10:54   ` Jiri Olsa
  2015-08-07 15:38   ` Arnaldo Carvalho de Melo
  1 sibling, 0 replies; 20+ messages in thread
From: Jiri Olsa @ 2015-08-07 10:54 UTC (permalink / raw)
  To: kan.liang; +Cc: acme, jolsa, namhyung, ak, linux-kernel

On Thu, Aug 06, 2015 at 03:44:53PM -0400, kan.liang@intel.com wrote:

SNIP

>             |
>             ---f3
>                |
>                |--0.53%-- f2
>                |          f1
>                |          main
>                |          __libc_start_main
>                |
>                |--0.01%-- f1
>                |          main
>                |          __libc_start_main
>                 --99.46%-- [...]
>     97.63%     0.03%  tchain_edit  tchain_edit       [.] f2
>             |
>             ---f2
>                f1
>                main
>                __libc_start_main
> 
>      7.13%     0.03%  tchain_edit  [kernel.vmlinux]  [k] do_nmi
>             |
>             ---do_nmi
>                end_repeat_nmi
>                f3
>                f2
>                f1
>                main
>                __libc_start_main
> 
> Signed-off-by: Kan Liang <kan.liang@intel.com>
> ---
> 
> Changes since V8:
>  - Same as global dump_size caculation, round_up dump_size for per-event dump_size.

Acked-by: Jiri Olsa <jolsa@kernel.org>

thanks,
jirka

^ permalink raw reply	[flat|nested] 20+ messages in thread

* Re: [PATCH RFC V9 3/3] perf,tests: Add tests to callgraph and time parse
  2015-08-06 19:44 ` [PATCH RFC V9 3/3] perf,tests: Add tests to callgraph and time parse kan.liang
@ 2015-08-07 10:54   ` Jiri Olsa
  0 siblings, 0 replies; 20+ messages in thread
From: Jiri Olsa @ 2015-08-07 10:54 UTC (permalink / raw)
  To: kan.liang; +Cc: acme, jolsa, namhyung, ak, linux-kernel

On Thu, Aug 06, 2015 at 03:44:54PM -0400, kan.liang@intel.com wrote:
> From: Kan Liang <kan.liang@intel.com>
> 
> Add tests in tests/parse-events.c to check call-graph and time option
> 
> Signed-off-by: Kan Liang <kan.liang@intel.com>

Acked-by: Jiri Olsa <jolsa@kernel.org>

thanks,
jirka

^ permalink raw reply	[flat|nested] 20+ messages in thread

* Re: [PATCH] perf tools: Move perf_counts struct and functions into separate object
  2015-08-07 10:51 ` [PATCH] perf tools: Move perf_counts struct and functions into separate object Jiri Olsa
@ 2015-08-07 13:14   ` Arnaldo Carvalho de Melo
  2015-08-12 12:28   ` [tip:perf/core] perf stat: " tip-bot for Jiri Olsa
  1 sibling, 0 replies; 20+ messages in thread
From: Arnaldo Carvalho de Melo @ 2015-08-07 13:14 UTC (permalink / raw)
  To: Jiri Olsa; +Cc: kan.liang, jolsa, namhyung, ak, linux-kernel

Em Fri, Aug 07, 2015 at 12:51:03PM +0200, Jiri Olsa escreveu:
> On Thu, Aug 06, 2015 at 03:44:52PM -0400, kan.liang@intel.com wrote:
> > From: Kan Liang <kan.liang@intel.com>
> > 
> > Move callchain option parse related code to util.c
> 
> little nore about the reason would be nice ;-)
> 
> looks ok, but the python test is still failing,
> the reason is the perf_counts struct objects and functions
> bindings to util/stat.c object
> 
> Arnaldo,
> I separated the 'struct perf_counts' into util/counts.c,
> because I recall you did not want them in evsel.c,
> please check attached patch.. it makes the python test
> pass again (on top of Kan's change).

Ok, will apply, try Kan's patch, add the extra comments about why the
move from callchain to util.c is needed, test the whole shebang, ship to
Ingo...

- Arnaldo
 
> thanks,
> jirka
> 
> 
> ---
> Moving perf_counts struct and functions into separate object,
> so we could remove stat.c object dependency from python build.
> 
> It makes the python code to be built properly, because now it
> fails to load due to missing stat-shadow.c object dependency.
> 
> Signed-off-by: Jiri Olsa <jolsa@kernel.org>
> ---
> diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c
> index a054ddc0b2a0..7aa039bd379a 100644
> --- a/tools/perf/builtin-stat.c
> +++ b/tools/perf/builtin-stat.c
> @@ -58,6 +58,7 @@
>  #include "util/cpumap.h"
>  #include "util/thread.h"
>  #include "util/thread_map.h"
> +#include "util/counts.h"
>  
>  #include <stdlib.h>
>  #include <sys/prctl.h>
> diff --git a/tools/perf/util/Build b/tools/perf/util/Build
> index a1e5168dc1fb..4ca481cd38e6 100644
> --- a/tools/perf/util/Build
> +++ b/tools/perf/util/Build
> @@ -67,6 +67,7 @@ libperf-y += target.o
>  libperf-y += rblist.o
>  libperf-y += intlist.o
>  libperf-y += vdso.o
> +libperf-y += counts.o
>  libperf-y += stat.o
>  libperf-y += stat-shadow.o
>  libperf-y += record.o
> diff --git a/tools/perf/util/counts.c b/tools/perf/util/counts.c
> new file mode 100644
> index 000000000000..e3fde313deb2
> --- /dev/null
> +++ b/tools/perf/util/counts.c
> @@ -0,0 +1,52 @@
> +#include <stdlib.h>
> +#include "evsel.h"
> +#include "counts.h"
> +
> +struct perf_counts *perf_counts__new(int ncpus, int nthreads)
> +{
> +	struct perf_counts *counts = zalloc(sizeof(*counts));
> +
> +	if (counts) {
> +		struct xyarray *values;
> +
> +		values = xyarray__new(ncpus, nthreads, sizeof(struct perf_counts_values));
> +		if (!values) {
> +			free(counts);
> +			return NULL;
> +		}
> +
> +		counts->values = values;
> +	}
> +
> +	return counts;
> +}
> +
> +void perf_counts__delete(struct perf_counts *counts)
> +{
> +	if (counts) {
> +		xyarray__delete(counts->values);
> +		free(counts);
> +	}
> +}
> +
> +static void perf_counts__reset(struct perf_counts *counts)
> +{
> +	xyarray__reset(counts->values);
> +}
> +
> +void perf_evsel__reset_counts(struct perf_evsel *evsel)
> +{
> +	perf_counts__reset(evsel->counts);
> +}
> +
> +int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads)
> +{
> +	evsel->counts = perf_counts__new(ncpus, nthreads);
> +	return evsel->counts != NULL ? 0 : -ENOMEM;
> +}
> +
> +void perf_evsel__free_counts(struct perf_evsel *evsel)
> +{
> +	perf_counts__delete(evsel->counts);
> +	evsel->counts = NULL;
> +}
> diff --git a/tools/perf/util/counts.h b/tools/perf/util/counts.h
> new file mode 100644
> index 000000000000..34d8baaf558a
> --- /dev/null
> +++ b/tools/perf/util/counts.h
> @@ -0,0 +1,37 @@
> +#ifndef __PERF_COUNTS_H
> +#define __PERF_COUNTS_H
> +
> +#include "xyarray.h"
> +
> +struct perf_counts_values {
> +	union {
> +		struct {
> +			u64 val;
> +			u64 ena;
> +			u64 run;
> +		};
> +		u64 values[3];
> +	};
> +};
> +
> +struct perf_counts {
> +	s8			  scaled;
> +	struct perf_counts_values aggr;
> +	struct xyarray		  *values;
> +};
> +
> +
> +static inline struct perf_counts_values*
> +perf_counts(struct perf_counts *counts, int cpu, int thread)
> +{
> +	return xyarray__entry(counts->values, cpu, thread);
> +}
> +
> +struct perf_counts *perf_counts__new(int ncpus, int nthreads);
> +void perf_counts__delete(struct perf_counts *counts);
> +
> +void perf_evsel__reset_counts(struct perf_evsel *evsel);
> +int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads);
> +void perf_evsel__free_counts(struct perf_evsel *evsel);
> +
> +#endif /* __PERF_COUNTS_H */
> diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h
> index 09a3022fa2c6..0ca0002db99a 100644
> --- a/tools/perf/util/evsel.h
> +++ b/tools/perf/util/evsel.h
> @@ -9,7 +9,7 @@
>  #include "xyarray.h"
>  #include "symbol.h"
>  #include "cpumap.h"
> -#include "stat.h"
> +#include "counts.h"
>  
>  struct perf_evsel;
>  
> diff --git a/tools/perf/util/python-ext-sources b/tools/perf/util/python-ext-sources
> index 0766d98c5da5..51be28b1bca2 100644
> --- a/tools/perf/util/python-ext-sources
> +++ b/tools/perf/util/python-ext-sources
> @@ -16,7 +16,7 @@ util/util.c
>  util/xyarray.c
>  util/cgroup.c
>  util/rblist.c
> -util/stat.c
> +util/counts.c
>  util/strlist.c
>  util/trace-event.c
>  ../lib/rbtree.c
> diff --git a/tools/perf/util/stat.c b/tools/perf/util/stat.c
> index c5c709cdc3ce..415c359de465 100644
> --- a/tools/perf/util/stat.c
> +++ b/tools/perf/util/stat.c
> @@ -97,55 +97,6 @@ void perf_stat_evsel_id_init(struct perf_evsel *evsel)
>  	}
>  }
>  
> -struct perf_counts *perf_counts__new(int ncpus, int nthreads)
> -{
> -	struct perf_counts *counts = zalloc(sizeof(*counts));
> -
> -	if (counts) {
> -		struct xyarray *values;
> -
> -		values = xyarray__new(ncpus, nthreads, sizeof(struct perf_counts_values));
> -		if (!values) {
> -			free(counts);
> -			return NULL;
> -		}
> -
> -		counts->values = values;
> -	}
> -
> -	return counts;
> -}
> -
> -void perf_counts__delete(struct perf_counts *counts)
> -{
> -	if (counts) {
> -		xyarray__delete(counts->values);
> -		free(counts);
> -	}
> -}
> -
> -static void perf_counts__reset(struct perf_counts *counts)
> -{
> -	xyarray__reset(counts->values);
> -}
> -
> -void perf_evsel__reset_counts(struct perf_evsel *evsel)
> -{
> -	perf_counts__reset(evsel->counts);
> -}
> -
> -int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads)
> -{
> -	evsel->counts = perf_counts__new(ncpus, nthreads);
> -	return evsel->counts != NULL ? 0 : -ENOMEM;
> -}
> -
> -void perf_evsel__free_counts(struct perf_evsel *evsel)
> -{
> -	perf_counts__delete(evsel->counts);
> -	evsel->counts = NULL;
> -}
> -
>  void perf_evsel__reset_stat_priv(struct perf_evsel *evsel)
>  {
>  	int i;
> diff --git a/tools/perf/util/stat.h b/tools/perf/util/stat.h
> index 0b897b083682..62448c8175d3 100644
> --- a/tools/perf/util/stat.h
> +++ b/tools/perf/util/stat.h
> @@ -33,23 +33,6 @@ enum aggr_mode {
>  	AGGR_THREAD,
>  };
>  
> -struct perf_counts_values {
> -	union {
> -		struct {
> -			u64 val;
> -			u64 ena;
> -			u64 run;
> -		};
> -		u64 values[3];
> -	};
> -};
> -
> -struct perf_counts {
> -	s8			  scaled;
> -	struct perf_counts_values aggr;
> -	struct xyarray		  *values;
> -};
> -
>  struct perf_stat_config {
>  	enum aggr_mode	aggr_mode;
>  	bool		scale;
> @@ -57,12 +40,6 @@ struct perf_stat_config {
>  	unsigned int	interval;
>  };
>  
> -static inline struct perf_counts_values*
> -perf_counts(struct perf_counts *counts, int cpu, int thread)
> -{
> -	return xyarray__entry(counts->values, cpu, thread);
> -}
> -
>  void update_stats(struct stats *stats, u64 val);
>  double avg_stats(struct stats *stats);
>  double stddev_stats(struct stats *stats);
> @@ -96,13 +73,6 @@ void perf_stat__update_shadow_stats(struct perf_evsel *counter, u64 *count,
>  void perf_stat__print_shadow_stats(FILE *out, struct perf_evsel *evsel,
>  				   double avg, int cpu, enum aggr_mode aggr);
>  
> -struct perf_counts *perf_counts__new(int ncpus, int nthreads);
> -void perf_counts__delete(struct perf_counts *counts);
> -
> -void perf_evsel__reset_counts(struct perf_evsel *evsel);
> -int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads);
> -void perf_evsel__free_counts(struct perf_evsel *evsel);
> -
>  void perf_evsel__reset_stat_priv(struct perf_evsel *evsel);
>  int perf_evsel__alloc_stat_priv(struct perf_evsel *evsel);
>  void perf_evsel__free_stat_priv(struct perf_evsel *evsel);

^ permalink raw reply	[flat|nested] 20+ messages in thread

* Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support
  2015-08-06 19:44 ` [PATCH RFC V9 2/3] perf,tools: per-event callgraph support kan.liang
  2015-08-07 10:54   ` Jiri Olsa
@ 2015-08-07 15:38   ` Arnaldo Carvalho de Melo
  2015-08-07 15:49     ` Arnaldo Carvalho de Melo
  2015-08-08 17:12     ` [PATCH] perf tools: Unset perf_event_attr::freq when period term is set Jiri Olsa
  1 sibling, 2 replies; 20+ messages in thread
From: Arnaldo Carvalho de Melo @ 2015-08-07 15:38 UTC (permalink / raw)
  To: kan.liang; +Cc: jolsa, namhyung, ak, linux-kernel

Em Thu, Aug 06, 2015 at 03:44:53PM -0400, kan.liang@intel.com escreveu:
> From: Kan Liang <kan.liang@intel.com>
> Here are some examples and test results.
> 
> 1. Comparing the elapsed time and perf.data size from "kernbench -M -H".
> 
>  The test command for FULL callgraph and time support.
>    "perf record -e
>    '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}'
>    --call-graph fp --time"

Jiri, while testing this I noticed that the message for EINVAL when
using the cpu// syntax (per-event settings) is cryptic:

  [root@zoo ~]# perf record -e 'cpu/cpu-cycles,call-graph=fp,time,period=100000/p' ls
  Error:
  The sys_perf_event_open() syscall returned with 22 (Invalid argument) for event (cpu/cpu-cycles,call-graph=fp,time,period=100000/p).
  /bin/dmesg may provide additional information.
  No CONFIG_PERF_EVENTS=y kernel support configured?

Whereas if we use -F, it is much, much clearer, telling the user exactly
what is failing and what needs to be done to make it work:

  [root@zoo ~]# perf record -F 100000 -e cpu/cpu-cycles/ usleep 1
  Maximum frequency rate (25000) reached.
  Please use -F freq option with lower value or consider
  tweaking /proc/sys/kernel/perf_event_max_sample_rate.
  [root@zoo ~]# 

Hope this is something easy to wire up, given your event parsing kung foo
skillz...

;-)

- Arnaldo

^ permalink raw reply	[flat|nested] 20+ messages in thread

* Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support
  2015-08-07 15:38   ` Arnaldo Carvalho de Melo
@ 2015-08-07 15:49     ` Arnaldo Carvalho de Melo
  2015-08-08 16:45       ` Jiri Olsa
  2015-08-10 12:56       ` Liang, Kan
  2015-08-08 17:12     ` [PATCH] perf tools: Unset perf_event_attr::freq when period term is set Jiri Olsa
  1 sibling, 2 replies; 20+ messages in thread
From: Arnaldo Carvalho de Melo @ 2015-08-07 15:49 UTC (permalink / raw)
  To: Kan Liang; +Cc: Jiri Olsa, Namhyung Kim, Andi Kleen, linux-kernel

Em Fri, Aug 07, 2015 at 12:38:43PM -0300, Arnaldo Carvalho de Melo escreveu:
> Em Thu, Aug 06, 2015 at 03:44:53PM -0400, kan.liang@intel.com escreveu:
> > From: Kan Liang <kan.liang@intel.com>
> > Here are some examples and test results.
> > 
> > 1. Comparing the elapsed time and perf.data size from "kernbench -M -H".
> > 
> >  The test command for FULL callgraph and time support.
> >    "perf record -e
> >    '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}'
> >    --call-graph fp --time"

Some other oddity I noticed while testing:

For:

[root@zoo ~]# perf report --header-only
# ========
# captured on: Fri Aug  7 12:39:13 2015
# hostname : zoo
# os release : 4.2.0-rc5+
# perf version : 4.2.rc4.g1cd951
# arch : x86_64
# nrcpus online : 4
# nrcpus avail : 4
# cpudesc : Intel(R) Core(TM) i7-3667U CPU @ 2.00GHz
# cpuid : GenuineIntel,6,58,9
# total memory : 8080692 kB
# cmdline : /home/acme/bin/perf record -e {cpu/cpu-cycles,call-graph=fp,time,period=10000/pp,cpu/instructions,call-graph=no,time=0,period=20000/p} -a 
# event : name = cpu/cpu-cycles,call-graph=fp,time,period=10000/pp, , id = { 38537, 38538, 38539, 38540 }, type = 4, size = 112, config = 0x3c, { sample_perio
# event : name = cpu/instructions,call-graph=no,time=0,period=20000/p, , id = { 38541, 38542, 38543, 38544 }, type = 4, size = 112, config = 0xc0, { sample_pe
# HEADER_CPU_TOPOLOGY info available, use -I to display
# HEADER_NUMA_TOPOLOGY info available, use -I to display
# pmu mappings: cpu = 4, software = 1, power = 7, uncore_imc = 8, tracepoint = 2, uncore_cbox_0 = 9, uncore_cbox_1 = 10, breakpoint = 5
# group: {cpu/cpu-cycles,call-graph=fp,time,period=10000/pp,cpu/instructions,call-graph=no,time=0,period=20000/p}
# ========
#

I get:

Samples: 2K of event 'cpu/instructions,call-graph=no,time=0,period=20000/p', Event count (approx.): 46956518
  Children      Self  Command          Shared Object Symbol                  ◆
-   67.56%     0.00%  qemu-system-x86  [unknown]     [.] 0xad5e258d4c544155  ▒
     0xad5e258d4c544155                                                      ▒
-   67.56%     0.00%  qemu-system-x86  libc-2.20.so  [.] __libc_start_main   ▒
     __libc_start_main                                                       ▒
     0xad5e258d4c544155                                                      ▒
-   67.56%     0.00%  qemu-system-x86  perf          [.] main                ▒
     main                                                                    ▒
     __libc_start_main                                                       ▒
     0xad5e258d4c544155                                                      ▒
-   67.56%     0.00%  qemu-system-x86  perf          [.] run_builtin         ▒
     run_builtin                                                             ▒
     main                                                                    ▒
     __libc_start_main                                                       ▒

This is in the 'perf report' TUI, why, for an event with 'callgraph=no', we
get callchains? How come?

If I try it with the --stdio, well, its there as well:

# Samples: 2K of event 'cpu/instructions,call-graph=no,time=0,period=20000/p'
# Event count (approx.): 46956518
#
# Children      Self  Command          Shared Object               Symbol                                        
# ........  ........  ...............  ..........................  ..............................................
#
    67.56%     0.00%  qemu-system-x86  [unknown]                   [.] 0xad5e258d4c544155                        
            |
            ---0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  libc-2.20.so                [.] __libc_start_main                         
            |
            ---__libc_start_main
               0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  perf                        [.] main                                      
            |
            ---main
               __libc_start_main
               0xad5e258d4c544155

    67.56%     0.00%  qemu-system-x86  perf                        [.] run_builtin                               
            |
            ---run_builtin
               main
               __libc_start_main
               0xad5e258d4c544155

Can you take a look? What I have, i.e. the patches from Jiri and Kan
moving stuff out of the way of the python binding and this patch is at
the tmp.perf/core branch in my git tree:

  git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git

- Arnaldo

 
> Jiri, while testing this I noticed that the message for EINVAL when
> using the cpu// syntax (per-event settings) is cryptic:
> 
>   [root@zoo ~]# perf record -e 'cpu/cpu-cycles,call-graph=fp,time,period=100000/p' ls
>   Error:
>   The sys_perf_event_open() syscall returned with 22 (Invalid argument) for event (cpu/cpu-cycles,call-graph=fp,time,period=100000/p).
>   /bin/dmesg may provide additional information.
>   No CONFIG_PERF_EVENTS=y kernel support configured?
> 
> Whereas if we use -F, it is much, much clearer, telling the user exactly
> what is failing and what needs to be done to make it work:
> 
>   [root@zoo ~]# perf record -F 100000 -e cpu/cpu-cycles/ usleep 1
>   Maximum frequency rate (25000) reached.
>   Please use -F freq option with lower value or consider
>   tweaking /proc/sys/kernel/perf_event_max_sample_rate.
>   [root@zoo ~]# 
> 
> Hope this is something easy to wire up, given your event parsing kung foo
> skillz...
> 
> ;-)
> 
> - Arnaldo

^ permalink raw reply	[flat|nested] 20+ messages in thread

* Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support
  2015-08-07 15:49     ` Arnaldo Carvalho de Melo
@ 2015-08-08 16:45       ` Jiri Olsa
  2015-08-08 17:35         ` Arnaldo Carvalho de Melo
  2015-08-10 13:46         ` Arnaldo Carvalho de Melo
  2015-08-10 12:56       ` Liang, Kan
  1 sibling, 2 replies; 20+ messages in thread
From: Jiri Olsa @ 2015-08-08 16:45 UTC (permalink / raw)
  To: Arnaldo Carvalho de Melo
  Cc: Kan Liang, Jiri Olsa, Namhyung Kim, Andi Kleen, linux-kernel

[-- Attachment #1: Type: text/plain, Size: 2657 bytes --]

On Fri, Aug 07, 2015 at 12:49:38PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Fri, Aug 07, 2015 at 12:38:43PM -0300, Arnaldo Carvalho de Melo escreveu:
> > Em Thu, Aug 06, 2015 at 03:44:53PM -0400, kan.liang@intel.com escreveu:
> > > From: Kan Liang <kan.liang@intel.com>
> > > Here are some examples and test results.
> > > 
> > > 1. Comparing the elapsed time and perf.data size from "kernbench -M -H".
> > > 
> > >  The test command for FULL callgraph and time support.
> > >    "perf record -e
> > >    '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}'
> > >    --call-graph fp --time"
> 
> Some other oddity I noticed while testing:
> 

SNIP

> This is in the 'perf report' TUI, why, for an event with 'callgraph=no', we
> get callchains? How come?
> 
> If I try it with the --stdio, well, its there as well:
> 
> # Samples: 2K of event 'cpu/instructions,call-graph=no,time=0,period=20000/p'
> # Event count (approx.): 46956518
> #
> # Children      Self  Command          Shared Object               Symbol                                        
> # ........  ........  ...............  ..........................  ..............................................
> #
>     67.56%     0.00%  qemu-system-x86  [unknown]                   [.] 0xad5e258d4c544155                        
>             |
>             ---0xad5e258d4c544155
> 
>     67.56%     0.00%  qemu-system-x86  libc-2.20.so                [.] __libc_start_main                         
>             |
>             ---__libc_start_main
>                0xad5e258d4c544155
> 
>     67.56%     0.00%  qemu-system-x86  perf                        [.] main                                      
>             |
>             ---main
>                __libc_start_main
>                0xad5e258d4c544155
> 
>     67.56%     0.00%  qemu-system-x86  perf                        [.] run_builtin                               
>             |
>             ---run_builtin
>                main
>                __libc_start_main
>                0xad5e258d4c544155
> 
> Can you take a look? What I have, i.e. the patches from Jiri and Kan
> moving stuff out of the way of the python binding and this patch is at
> the tmp.perf/core branch in my git tree:
> 
>   git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git

hum, tried this branch and got proper behaviour for both stdio and TUI,
attaching stdio output

the record command was:
  $ ./perf record -e 'cpu/cpu-cycles,call-graph=fp,time,period=10000/pp,cpu/instructions,call-graph=no,time=0,period=20000/p' ls

got the events strings from your report output.. what was different in your record command?

jirka

[-- Attachment #2: p --]
[-- Type: text/plain, Size: 162659 bytes --]

# To display the perf.data header info, please use --header/--header-only options.
#
#
# Total Lost Samples: 4
#
# Samples: 95  of event 'cpu/cpu-cycles,call-graph=fp,time,period=10000/pp'
# Event count (approx.): 7487606
#
# Children      Self  Command  Shared Object       Symbol                               
# ........  ........  .......  ..................  .....................................
#
    61.66%     0.00%  ls       [kernel.vmlinux]    [k] entry_SYSCALL_64_fastpath        
                 |
                 ---entry_SYSCALL_64_fastpath
                    |          
                    |--55.19%-- open64
                    |          0x686361632e6f732e
                    |          
                    |--21.81%-- __GI___libc_write
                    |          |          
                    |          |--24.74%-- 0x6e69746c69756220
                    |          |          
                    |          |--19.20%-- 0x909632e66666964
                    |          |          
                    |          |--18.35%-- 0x657461746f6e6e61
                    |          |          
                    |          |--16.05%-- 0x1dbdc70
                    |          |          
                    |          |--11.32%-- 0
                    |          |          
                    |           --10.34%-- 0x9096f2e61746164
                    |          
                    |--6.18%-- __getdents64
                    |          
                    |--5.00%-- mmap64
                    |          |          
                    |          |--50.39%-- _dl_check_all_versions
                    |          |          
                    |           --49.61%-- _dl_map_object
                    |                     0x213988
                    |                     0
                    |          
                    |--2.72%-- mprotect
                    |          dl_main
                    |          _dl_sysdep_start
                    |          
                    |--1.26%-- _exit
                    |          __run_exit_handlers
                    |          0
                    |          
                    |--0.92%-- __GI___munmap
                     --6.92%-- [...]

    34.03%     0.00%  ls       [unknown]           [.] 0x686361632e6f732e               
                 |
                 ---0x686361632e6f732e

    34.03%     0.00%  ls       ld-2.18.so          [.] open64                           
                 |
                 ---open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] sys_open                         
                 |
                 ---sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] do_sys_open                      
                 |
                 ---do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] do_filp_open                     
                 |
                 ---do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] path_openat                      
                 |
                 ---path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] link_path_walk                   
                 |
                 ---link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] inode_permission                 
                 |
                 ---inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    34.03%     0.00%  ls       [kernel.vmlinux]    [k] __inode_permission               
                 |
                 ---__inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] irq_work_interrupt               
                 |
                 ---irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] smp_irq_work_interrupt           
                 |
                 ---smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] irq_work_run                     
                 |
                 ---irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] irq_work_run_list                
                 |
                 ---irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] perf_duration_warn               
                 |
                 ---perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] printk                           
                 |
                 ---printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] vprintk_default                  
                 |
                 ---vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] vprintk_emit                     
                 |
                 ---vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.75%     0.00%  ls       [kernel.vmlinux]    [k] console_unlock                   
                 |
                 ---console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.43%     0.00%  ls       [kernel.vmlinux]    [k] call_console_drivers.constprop.27
                 |
                 ---call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.43%     0.00%  ls       [kernel.vmlinux]    [k] univ8250_console_write           
                 |
                 ---univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.43%     0.00%  ls       [kernel.vmlinux]    [k] uart_console_write               
                 |
                 ---uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    33.43%     0.00%  ls       [kernel.vmlinux]    [k] serial8250_console_putchar       
                 |
                 ---serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    28.96%     0.00%  ls       [kernel.vmlinux]    [k] wait_for_xmitr                   
                 |
                 ---wait_for_xmitr
                    serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    17.71%     0.00%  ls       libc-2.18.so        [.] __GI___libc_write                
                 |
                 ---__GI___libc_write
                    |          
                    |--16.81%-- 0x657461746f6e6e61
                    |          
                    |--10.79%-- 0x6e69746c69756220
                    |          
                    |--8.37%-- 0x909632e66666964
                    |          
                    |--7.00%-- 0x1dbdc70
                    |          
                    |--6.67%-- 0x96f2e68636e6562
                    |          
                    |--5.30%-- 0x2d6469646c697562
                    |          
                    |--4.93%-- 0
                    |          
                    |--4.51%-- 0x9096f2e61746164
                    |          
                    |--3.30%-- 0x6f2e7473696c7665
                     --32.32%-- [...]

    14.14%     0.41%  ls       [kernel.vmlinux]    [k] __const_udelay                   
                 |
                 ---__const_udelay
                    |          
                    |--97.12%-- wait_for_xmitr
                    |          serial8250_console_putchar
                    |          uart_console_write
                    |          univ8250_console_write
                    |          call_console_drivers.constprop.27
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --2.88%-- serial8250_console_putchar
                               uart_console_write
                               univ8250_console_write
                               call_console_drivers.constprop.27
                               console_unlock
                               vprintk_emit
                               vprintk_default
                               printk
                               perf_duration_warn
                               irq_work_run_list
                               irq_work_run
                               smp_irq_work_interrupt
                               irq_work_interrupt
                               __inode_permission
                               inode_permission
                               link_path_walk
                               path_openat
                               do_filp_open
                               do_sys_open
                               sys_open
                               entry_SYSCALL_64_fastpath
                               open64
                               0x686361632e6f732e

    13.75%    13.75%  ls       [kernel.vmlinux]    [k] io_serial_in                     
                 |
                 ---io_serial_in
                    wait_for_xmitr
                    serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] sys_write                        
                 |
                 ---sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] vfs_write                        
                 |
                 ---vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] __vfs_write                      
                 |
                 ---__vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] tty_write                        
                 |
                 ---tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    13.45%     0.00%  ls       [kernel.vmlinux]    [k] n_tty_write                      
                 |
                 ---n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--14.21%-- 0x6e69746c69756220
                    |          
                    |--11.03%-- 0x909632e66666964
                    |          
                    |--10.54%-- 0x657461746f6e6e61
                    |          
                    |--9.22%-- 0x1dbdc70
                    |          
                    |--6.50%-- 0
                    |          
                    |--5.94%-- 0x9096f2e61746164
                     --42.57%-- [...]

    12.39%    12.39%  ls       [kernel.vmlinux]    [k] delay_tsc                        
                 |
                 ---delay_tsc
                    |          
                    |--88.16%-- __const_udelay
                    |          wait_for_xmitr
                    |          serial8250_console_putchar
                    |          uart_console_write
                    |          univ8250_console_write
                    |          call_console_drivers.constprop.27
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --11.84%-- wait_for_xmitr
                               serial8250_console_putchar
                               uart_console_write
                               univ8250_console_write
                               call_console_drivers.constprop.27
                               console_unlock
                               vprintk_emit
                               vprintk_default
                               printk
                               perf_duration_warn
                               irq_work_run_list
                               irq_work_run
                               smp_irq_work_interrupt
                               irq_work_interrupt
                               __inode_permission
                               inode_permission
                               link_path_walk
                               path_openat
                               do_filp_open
                               do_sys_open
                               sys_open
                               entry_SYSCALL_64_fastpath
                               open64
                               0x686361632e6f732e

     9.88%     0.00%  ls       [unknown]           [.] 0x0000000000841f0f               
                 |
                 ---0x841f0f

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] pty_write                        
                 |
                 ---pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] tty_flip_buffer_push             
                 |
                 ---tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] tty_schedule_flip                
                 |
                 ---tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] queue_work_on                    
                 |
                 ---queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     0.00%  ls       [kernel.vmlinux]    [k] __queue_work                     
                 |
                 ---__queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     9.16%     1.30%  ls       [kernel.vmlinux]    [k] insert_work                      
                 |
                 ---insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--45.92%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--31.59%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--69.57%-- 0x657461746f6e6e61
                    |          |          
                    |           --30.43%-- 0x1dbdc70
                    |          
                     --22.49%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     8.41%     0.00%  ls       [unknown]           [.] 0000000000000000                 
                 |
                 ---0
                    |          
                    |--15.30%-- 0x632e74617473
                    |          
                    |--12.88%-- 0
                    |          
                    |--12.01%-- 0x72616863656d6974
                     --59.81%-- [...]

     7.92%     0.00%  ls       ls                  [.] 0xffffffffffc0e1cf               
                 |
                 ---0xe1cf
                    0x841f0f

     6.97%     0.00%  ls       [kernel.vmlinux]    [k] wake_up_worker                   
                 |
                 ---wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--37.80%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--32.63%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x657461746f6e6e61
                    |          
                     --29.57%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     6.97%     0.00%  ls       [kernel.vmlinux]    [k] wake_up_process                  
                 |
                 ---wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--37.80%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--32.63%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x657461746f6e6e61
                    |          
                     --29.57%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     6.97%     0.00%  ls       [kernel.vmlinux]    [k] try_to_wake_up                   
                 |
                 ---try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--37.80%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--32.63%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x657461746f6e6e61
                    |          
                     --29.57%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     6.97%     0.00%  ls       [kernel.vmlinux]    [k] ttwu_do_activate.constprop.92    
                 |
                 ---ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--37.80%-- n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--72.50%-- 0x6e69746c69756220
                    |          |          
                    |           --27.50%-- 0x909632e66666964
                    |          
                    |--32.63%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x657461746f6e6e61
                    |          
                     --29.57%-- tty_put_char
                               do_output_char
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write

     5.94%     5.94%  ls       libc-2.18.so        [.] get_next_seq                     
                 |
                 ---get_next_seq
                    0xe1cf
                    0x841f0f

     5.80%     0.00%  ls       [kernel.vmlinux]    [k] __schedule                       
                 |
                 ---__schedule
                    |          
                    |--73.77%-- preempt_schedule_common
                    |          _cond_resched
                    |          mutex_lock
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--28.65%-- 0
                    |          |          
                    |          |--26.18%-- 0x9096f2e61746164
                    |          |          
                    |          |--24.86%-- 0x909632e66666964
                    |          |          
                    |           --20.31%-- 0x1dbdc70
                    |          
                     --26.23%-- schedule
                               prepare_exit_to_usermode
                               syscall_return_slowpath
                               int_ret_from_sys_call
                               __GI___libc_write
                               |          
                               |--61.62%-- 0x2d6469646c697562
                               |          
                                --38.38%-- 0x6f2e7473696c7665

     5.75%     1.94%  ls       libc-2.18.so        [.] __getdents64                     
                 |
                 ---__getdents64
                    |          
                    |--33.68%-- 0x616863656d69742d
                     --66.32%-- [...]

     5.04%     0.00%  ls       [kernel.vmlinux]    [k] 0x000000007f47cb44               
                 |
                 ---0x27cb44
                    entry_SYSCALL_64_fastpath
                    |          
                    |--15.40%-- _exit
                    |          __run_exit_handlers
                    |          0
                     --84.60%-- [...]

     5.04%     0.00%  ls       [kernel.vmlinux]    [k] do_group_exit                    
                 |
                 ---do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath
                    |          
                    |--15.40%-- _exit
                    |          __run_exit_handlers
                    |          0
                     --84.60%-- [...]

     5.04%     0.78%  ls       [kernel.vmlinux]    [k] do_exit                          
                 |
                 ---do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath
                    |          
                    |--15.40%-- _exit
                    |          __run_exit_handlers
                    |          0
                     --84.60%-- [...]

     4.96%     0.00%  ls       [kernel.vmlinux]    [k] do_output_char                   
                 |
                 ---do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--28.59%-- 0x657461746f6e6e61
                    |          
                    |--12.51%-- 0x1dbdc70
                     --58.90%-- [...]

     4.79%     4.79%  ls       [kernel.vmlinux]    [k] native_write_msr_safe            
                 |
                 ---native_write_msr_safe
                    |          
                    |--37.79%-- __intel_pmu_disable_all
                    |          intel_pmu_disable_all
                    |          x86_pmu_disable
                    |          perf_pmu_disable
                    |          ctx_sched_out
                    |          __perf_event_task_sched_out
                    |          __schedule
                    |          |          
                    |          |--51.77%-- schedule
                    |          |          prepare_exit_to_usermode
                    |          |          syscall_return_slowpath
                    |          |          int_ret_from_sys_call
                    |          |          __GI___libc_write
                    |          |          0x2d6469646c697562
                    |          |          
                    |           --48.23%-- preempt_schedule_common
                    |                     _cond_resched
                    |                     mutex_lock
                    |                     n_tty_write
                    |                     tty_write
                    |                     __vfs_write
                    |                     vfs_write
                    |                     sys_write
                    |                     entry_SYSCALL_64_fastpath
                    |                     __GI___libc_write
                    |                     0
                    |          
                    |--32.55%-- __GI___libc_write
                    |          0x657461746f6e6e61
                    |          
                    |--29.58%-- __intel_pmu_enable_all.isra.9
                    |          intel_pmu_enable_all
                    |          x86_pmu_enable
                    |          perf_pmu_enable
                    |          |          
                    |          |--56.32%-- perf_event_context_sched_in
                    |          |          |          
                    |          |          |--100.00%-- __perf_event_task_sched_in
                    |          |          |          finish_task_switch
                    |          |          |          __schedule
                    |          |          |          preempt_schedule_common
                    |          |          |          _cond_resched
                    |          |          |          mutex_lock
                    |          |          |          n_tty_write
                    |          |          |          tty_write
                    |          |          |          __vfs_write
                    |          |          |          vfs_write
                    |          |          |          sys_write
                    |          |          |          entry_SYSCALL_64_fastpath
                    |          |          |          __GI___libc_write
                    |          |          |          0x9096f2e61746164
                    |          |           --0.00%-- [...]
                    |          |          
                    |           --43.68%-- perf_pmu_sched_task
                    |                     __perf_event_task_sched_out
                    |                     __schedule
                    |                     preempt_schedule_common
                    |                     _cond_resched
                    |                     mutex_lock
                    |                     n_tty_write
                    |                     tty_write
                    |                     __vfs_write
                    |                     vfs_write
                    |                     sys_write
                    |                     entry_SYSCALL_64_fastpath
                    |                     __GI___libc_write
                    |                     0x1dbdc70
                     --0.08%-- [...]

     4.28%     0.00%  ls       [kernel.vmlinux]    [k] mutex_lock                       
                 |
                 ---mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--20.42%-- 0
                    |          
                    |--18.66%-- 0x9096f2e61746164
                    |          
                    |--17.71%-- 0x909632e66666964
                    |          
                    |--14.47%-- 0x1dbdc70
                     --28.74%-- [...]

     4.28%     0.00%  ls       [kernel.vmlinux]    [k] _cond_resched                    
                 |
                 ---_cond_resched
                    mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--20.42%-- 0
                    |          
                    |--18.66%-- 0x9096f2e61746164
                    |          
                    |--17.71%-- 0x909632e66666964
                    |          
                    |--14.47%-- 0x1dbdc70
                     --28.74%-- [...]

     4.28%     0.00%  ls       [kernel.vmlinux]    [k] preempt_schedule_common          
                 |
                 ---preempt_schedule_common
                    _cond_resched
                    mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--20.42%-- 0
                    |          
                    |--18.66%-- 0x9096f2e61746164
                    |          
                    |--17.71%-- 0x909632e66666964
                    |          
                    |--14.47%-- 0x1dbdc70
                     --28.74%-- [...]

     4.06%     4.06%  ls       [kernel.vmlinux]    [k] io_serial_out                    
                 |
                 ---io_serial_out
                    serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     4.04%     4.04%  ls       ls                  [.] 0x0000000000006686               
                 |
                 ---0x6686

     4.04%     0.00%  ls       ls                  [.] 0xffffffffffc06686               
                 |
                 ---0x6686

     3.81%     0.00%  ls       [kernel.vmlinux]    [k] sys_getdents                     
                 |
                 ---sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.81%     0.00%  ls       [kernel.vmlinux]    [k] iterate_dir                      
                 |
                 ---iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.81%     0.00%  ls       [ext4]              [k] ext4_readdir                     
                 |
                 ---ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.81%     0.00%  ls       [ext4]              [k] ext4_htree_fill_tree             
                 |
                 ---ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.81%     0.00%  ls       [ext4]              [k] htree_dirblock_to_tree           
                 |
                 ---htree_dirblock_to_tree
                    ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     3.64%     0.00%  ls       [kernel.vmlinux]    [k] activate_task                    
                 |
                 ---activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--56.57%-- tty_put_char
                    |          do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                    |--23.55%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                     --19.88%-- n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x909632e66666964

     3.64%     0.00%  ls       [kernel.vmlinux]    [k] enqueue_task                     
                 |
                 ---enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--56.57%-- tty_put_char
                    |          do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                    |--23.55%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                     --19.88%-- n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x909632e66666964

     3.64%     0.00%  ls       [kernel.vmlinux]    [k] enqueue_task_fair                
                 |
                 ---enqueue_task_fair
                    enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--56.57%-- tty_put_char
                    |          do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                    |--23.55%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                     --19.88%-- n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x909632e66666964

     3.38%     0.00%  ls       ld-2.18.so          [.] _dl_map_object                   
                 |
                 ---_dl_map_object
                    |          
                    |--45.27%-- 0x21eb10
                    |          0
                    |          
                    |--45.27%-- 0x213988
                    |          0
                    |          
                     --9.45%-- 0x21e640
                               0

     3.27%     0.00%  ls       ld-2.18.so          [.] _dl_sysdep_start                 
                 |
                 ---_dl_sysdep_start

     3.27%     0.00%  ls       ld-2.18.so          [.] dl_main                          
                 |
                 ---dl_main
                    _dl_sysdep_start

     3.08%     0.00%  ls       ld-2.18.so          [.] mmap64                           
                 |
                 ---mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] sys_mmap                         
                 |
                 ---sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] sys_mmap_pgoff                   
                 |
                 ---sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] vm_mmap_pgoff                    
                 |
                 ---vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] do_mmap_pgoff                    
                 |
                 ---do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.08%     0.00%  ls       [kernel.vmlinux]    [k] mmap_region                      
                 |
                 ---mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    |          
                    |--50.39%-- _dl_check_all_versions
                    |          
                     --49.61%-- _dl_map_object
                               0x213988
                               0

     3.03%     0.00%  ls       [kernel.vmlinux]    [k] mmput                            
                 |
                 ---mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     3.03%     0.00%  ls       [kernel.vmlinux]    [k] exit_mmap                        
                 |
                 ---exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     2.98%     0.00%  ls       [unknown]           [.] 0x657461746f6e6e61               
                 |
                 ---0x657461746f6e6e61

     2.79%     0.64%  ls       [kernel.vmlinux]    [k] finish_task_switch               
                 |
                 ---finish_task_switch
                    __schedule
                    preempt_schedule_common
                    _cond_resched
                    mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--28.65%-- 0x9096f2e61746164
                    |          
                    |--27.20%-- 0x909632e66666964
                     --44.14%-- [...]

     2.70%     0.00%  ls       [kernel.vmlinux]    [k] int_ret_from_sys_call            
                 |
                 ---int_ret_from_sys_call
                    __GI___libc_write
                    |          
                    |--43.71%-- 0x96f2e68636e6562
                    |          
                    |--34.69%-- 0x2d6469646c697562
                    |          
                     --21.61%-- 0x6f2e7473696c7665

     2.70%     0.00%  ls       [kernel.vmlinux]    [k] syscall_return_slowpath          
                 |
                 ---syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    |          
                    |--43.71%-- 0x96f2e68636e6562
                    |          
                    |--34.69%-- 0x2d6469646c697562
                    |          
                     --21.61%-- 0x6f2e7473696c7665

     2.70%     1.18%  ls       [kernel.vmlinux]    [k] prepare_exit_to_usermode         
                 |
                 ---prepare_exit_to_usermode
                    syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    |          
                    |--43.71%-- 0x96f2e68636e6562
                    |          
                    |--34.69%-- 0x2d6469646c697562
                    |          
                     --21.61%-- 0x6f2e7473696c7665

     2.43%     0.00%  ls       [kernel.vmlinux]    [k] __perf_event_task_sched_out      
                 |
                 ---__perf_event_task_sched_out
                    __schedule
                    |          
                    |--61.42%-- preempt_schedule_common
                    |          _cond_resched
                    |          mutex_lock
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |          |--58.52%-- 0
                    |          |          
                    |           --41.48%-- 0x1dbdc70
                    |          
                     --38.58%-- schedule
                               prepare_exit_to_usermode
                               syscall_return_slowpath
                               int_ret_from_sys_call
                               __GI___libc_write
                               0x2d6469646c697562

     2.30%     0.00%  ls       ls                  [.] 0xffffffffffc06dcb               
                 |
                 ---0x6dcb
                    0
                    |          
                    |--56.03%-- 0x632e74617473
                    |          
                     --43.97%-- 0x72616863656d6974

     2.26%     1.63%  ls       ld-2.18.so          [.] do_lookup_x                      
                 |
                 ---do_lookup_x

     2.16%     1.54%  ls       [kernel.vmlinux]    [k] nmi                              
                 |
                 ---nmi
                    |          
                    |--63.15%-- __const_udelay
                    |          wait_for_xmitr
                    |          serial8250_console_putchar
                    |          uart_console_write
                    |          univ8250_console_write
                    |          call_console_drivers.constprop.27
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                    |--29.07%-- do_lookup_x
                    |          
                    |--5.52%-- end_repeat_nmi
                    |          scheduler_tick
                    |          update_process_times
                    |          tick_sched_handle.isra.17
                    |          tick_sched_timer
                    |          __hrtimer_run_queues
                    |          hrtimer_interrupt
                    |          local_apic_timer_interrupt
                    |          smp_apic_timer_interrupt
                    |          apic_timer_interrupt
                    |          setup_new_exec
                    |          load_elf_binary
                    |          search_binary_handler
                    |          do_execveat_common.isra.32
                    |          sys_execve
                    |          return_from_execve
                    |          0x7f9abb498a77
                    |          
                    |--1.10%-- nmi_handle
                    |          default_do_nmi
                    |          do_nmi
                    |          end_repeat_nmi
                    |          scheduler_tick
                    |          update_process_times
                    |          tick_sched_handle.isra.17
                    |          tick_sched_timer
                    |          __hrtimer_run_queues
                    |          hrtimer_interrupt
                    |          local_apic_timer_interrupt
                    |          smp_apic_timer_interrupt
                    |          apic_timer_interrupt
                    |          setup_new_exec
                    |          load_elf_binary
                    |          search_binary_handler
                    |          do_execveat_common.isra.32
                    |          sys_execve
                    |          return_from_execve
                    |          0x7f9abb498a77
                    |          
                    |--0.63%-- perf_event_nmi_handler
                    |          nmi_handle
                    |          default_do_nmi
                    |          do_nmi
                    |          end_repeat_nmi
                    |          scheduler_tick
                    |          update_process_times
                    |          tick_sched_handle.isra.17
                    |          tick_sched_timer
                    |          __hrtimer_run_queues
                    |          hrtimer_interrupt
                    |          local_apic_timer_interrupt
                    |          smp_apic_timer_interrupt
                    |          apic_timer_interrupt
                    |          setup_new_exec
                    |          load_elf_binary
                    |          search_binary_handler
                    |          do_execveat_common.isra.32
                    |          sys_execve
                    |          return_from_execve
                    |          0x7f9abb498a77
                    |          
                     --0.52%-- scheduler_tick
                               update_process_times
                               tick_sched_handle.isra.17
                               tick_sched_timer
                               __hrtimer_run_queues
                               hrtimer_interrupt
                               local_apic_timer_interrupt
                               smp_apic_timer_interrupt
                               apic_timer_interrupt
                               setup_new_exec
                               load_elf_binary
                               search_binary_handler
                               do_execveat_common.isra.32
                               sys_execve
                               return_from_execve
                               0x7f9abb498a77

     2.15%     0.00%  ls       [kernel.vmlinux]    [k] __perf_event_task_sched_in       
                 |
                 ---__perf_event_task_sched_in
                    finish_task_switch
                    __schedule
                    preempt_schedule_common
                    _cond_resched
                    mutex_lock
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    |          
                    |--37.19%-- 0x9096f2e61746164
                    |          
                    |--35.31%-- 0x909632e66666964
                     --27.50%-- [...]

     2.06%     0.00%  ls       [kernel.vmlinux]    [k] tty_put_char                     
                 |
                 ---tty_put_char
                    do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write

     2.06%     2.06%  ls       [kernel.vmlinux]    [k] update_rq_runnable_avg           
                 |
                 ---update_rq_runnable_avg
                    enqueue_task_fair
                    enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    tty_put_char
                    do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write

     1.98%     1.98%  ls       libc-2.18.so        [.] __strcoll_l                      
                 |
                 ---__strcoll_l
                    0xe1cf
                    0x841f0f

     1.97%     1.35%  ls       [kernel.vmlinux]    [k] perf_pmu_sched_task              
                 |
                 ---perf_pmu_sched_task
                    |          
                    |--68.52%-- __perf_event_task_sched_in
                    |          finish_task_switch
                    |          __schedule
                    |          preempt_schedule_common
                    |          _cond_resched
                    |          mutex_lock
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          |          
                    |           --100.00%-- 0x909632e66666964
                    |          
                     --31.48%-- __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.95%     1.95%  ls       ls                  [.] 0x000000000000e270               
                 |
                 ---0xe270
                    0x841f0f

     1.95%     0.00%  ls       ls                  [.] 0xffffffffffc0e270               
                 |
                 ---0xe270
                    0x841f0f

     1.94%     0.00%  ls       [unknown]           [.] 0x616863656d69742d               
                 |
                 ---0x616863656d69742d

     1.93%     0.00%  ls       [kernel.vmlinux]    [k] unmap_vmas                       
                 |
                 ---unmap_vmas
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.93%     0.93%  ls       [kernel.vmlinux]    [k] unmap_single_vma                 
                 |
                 ---unmap_single_vma
                    unmap_vmas
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.93%     0.00%  ls       [ext4]              [k] ext4fs_dirhash                   
                 |
                 ---ext4fs_dirhash
                    htree_dirblock_to_tree
                    ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     1.93%     1.93%  ls       [ext4]              [k] str2hashbuf_signed               
                 |
                 ---str2hashbuf_signed
                    ext4fs_dirhash
                    htree_dirblock_to_tree
                    ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     1.91%     0.00%  ls       [unknown]           [.] 0x6e69746c69756220               
                 |
                 ---0x6e69746c69756220

     1.91%     0.00%  ls       [kernel.vmlinux]    [k] ttwu_do_wakeup                   
                 |
                 ---ttwu_do_wakeup
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x6e69746c69756220

     1.91%     0.00%  ls       [kernel.vmlinux]    [k] check_preempt_curr               
                 |
                 ---check_preempt_curr
                    ttwu_do_wakeup
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x6e69746c69756220

     1.91%     0.00%  ls       [kernel.vmlinux]    [k] check_preempt_wakeup             
                 |
                 ---check_preempt_wakeup
                    check_preempt_curr
                    ttwu_do_wakeup
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x6e69746c69756220

     1.91%     1.91%  ls       [kernel.vmlinux]    [k] set_next_buddy                   
                 |
                 ---set_next_buddy
                    check_preempt_wakeup
                    check_preempt_curr
                    ttwu_do_wakeup
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x6e69746c69756220

     1.89%     1.89%  ls       [ext4]              [k] ext4_htree_store_dirent          
                 |
                 ---ext4_htree_store_dirent
                    htree_dirblock_to_tree
                    ext4_htree_fill_tree
                    ext4_readdir
                    iterate_dir
                    sys_getdents
                    entry_SYSCALL_64_fastpath
                    __getdents64

     1.85%     1.85%  ls       libc-2.18.so        [.] __strcmp_sse2                    
                 |
                 ---__strcmp_sse2

     1.82%     0.00%  ls       [unknown]           [.] 0x6f74207367756220               
                 |
                 ---0x6f74207367756220

     1.82%     1.82%  ls       libc-2.18.so        [.] strlen                           
                 |
                 ---strlen
                    0x6f74207367756220

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] ctx_sched_out                    
                 |
                 ---ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] perf_pmu_disable                 
                 |
                 ---perf_pmu_disable
                    ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] x86_pmu_disable                  
                 |
                 ---x86_pmu_disable
                    perf_pmu_disable
                    ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] intel_pmu_disable_all            
                 |
                 ---intel_pmu_disable_all
                    x86_pmu_disable
                    perf_pmu_disable
                    ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.81%     0.00%  ls       [kernel.vmlinux]    [k] __intel_pmu_disable_all          
                 |
                 ---__intel_pmu_disable_all
                    intel_pmu_disable_all
                    x86_pmu_disable
                    perf_pmu_disable
                    ctx_sched_out
                    __perf_event_task_sched_out
                    __schedule
                    |          
                    |--51.77%-- schedule
                    |          prepare_exit_to_usermode
                    |          syscall_return_slowpath
                    |          int_ret_from_sys_call
                    |          __GI___libc_write
                    |          0x2d6469646c697562
                    |          
                     --48.23%-- preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0

     1.76%     0.00%  ls       [unknown]           [.] 0x00401f0fc3f30678               
                 |
                 ---0x401f0fc3f30678

     1.76%     1.76%  ls       libc-2.18.so        [.] _dl_addr                         
                 |
                 ---_dl_addr
                    0x401f0fc3f30678

     1.72%     0.00%  ls       ls                  [.] 0xffffffffffc06438               
                 |
                 ---0x6438

     1.72%     1.72%  ls       ls                  [.] 0x0000000000005b4d               
                 |
                 ---0x5b4d
                    0x6438

     1.72%     0.00%  ls       ls                  [.] 0xffffffffffc05b4d               
                 |
                 ---0x5b4d
                    0x6438

     1.70%     1.70%  ls       libpthread-2.18.so  [.] frame_dummy                      
                 |
                 ---frame_dummy

     1.68%     0.00%  ls       ld-2.18.so          [.] mprotect                         
                 |
                 ---mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     0.00%  ls       [kernel.vmlinux]    [k] sys_mprotect                     
                 |
                 ---sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     0.00%  ls       [kernel.vmlinux]    [k] mprotect_fixup                   
                 |
                 ---mprotect_fixup
                    sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     0.00%  ls       [kernel.vmlinux]    [k] split_vma                        
                 |
                 ---split_vma
                    mprotect_fixup
                    sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     0.00%  ls       [kernel.vmlinux]    [k] __split_vma.isra.34              
                 |
                 ---__split_vma.isra.34
                    split_vma
                    mprotect_fixup
                    sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.68%     1.68%  ls       [kernel.vmlinux]    [k] vma_interval_tree_remove         
                 |
                 ---vma_interval_tree_remove
                    __split_vma.isra.34
                    split_vma
                    mprotect_fixup
                    sys_mprotect
                    entry_SYSCALL_64_fastpath
                    mprotect
                    dl_main
                    _dl_sysdep_start

     1.59%     1.59%  ls       ld-2.18.so          [.] _dl_relocate_object              
                 |
                 ---_dl_relocate_object
                    dl_main
                    _dl_sysdep_start

     1.58%     0.86%  ls       [kernel.vmlinux]    [k] enqueue_entity                   
                 |
                 ---enqueue_entity
                    enqueue_task_fair
                    enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    |          
                    |--54.22%-- do_output_char
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          
                     --45.78%-- n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x909632e66666964

     1.55%     0.00%  ls       ld-2.18.so          [.] _dl_check_all_versions           
                 |
                 ---_dl_check_all_versions

     1.55%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_mmap                  
                 |
                 ---perf_event_mmap
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_check_all_versions

     1.55%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_aux                   
                 |
                 ---perf_event_aux
                    perf_event_mmap
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_check_all_versions

     1.55%     1.55%  ls       [kernel.vmlinux]    [k] perf_event_aux_ctx               
                 |
                 ---perf_event_aux_ctx
                    perf_event_aux
                    perf_event_mmap
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_check_all_versions

     1.53%     0.00%  ls       ld-2.18.so          [.] 0xffff804cabb57988               
                 |
                 ---0x213988
                    0

     1.53%     0.00%  ls       ld-2.18.so          [.] 0xffff804cabb62b10               
                 |
                 ---0x21eb10
                    0

     1.53%     0.00%  ls       ld-2.18.so          [.] memset                           
                 |
                 ---memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] page_fault                       
                 |
                 ---page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] do_page_fault                    
                 |
                 ---do_page_fault
                    page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] __do_page_fault                  
                 |
                 ---__do_page_fault
                    do_page_fault
                    page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] handle_mm_fault                  
                 |
                 ---handle_mm_fault
                    __do_page_fault
                    do_page_fault
                    page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     1.53%  ls       [kernel.vmlinux]    [k] put_page                         
                 |
                 ---put_page
                    handle_mm_fault
                    __do_page_fault
                    do_page_fault
                    page_fault
                    memset
                    _dl_map_object
                    0x21eb10
                    0

     1.53%     0.00%  ls       [kernel.vmlinux]    [k] vma_link                         
                 |
                 ---vma_link
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_map_object
                    0x213988
                    0

     1.53%     1.53%  ls       [kernel.vmlinux]    [k] vma_interval_tree_insert         
                 |
                 ---vma_interval_tree_insert
                    vma_link
                    mmap_region
                    do_mmap_pgoff
                    vm_mmap_pgoff
                    sys_mmap_pgoff
                    sys_mmap
                    entry_SYSCALL_64_fastpath
                    mmap64
                    _dl_map_object
                    0x213988
                    0

     1.52%     0.00%  ls       [kernel.vmlinux]    [k] schedule                         
                 |
                 ---schedule
                    prepare_exit_to_usermode
                    syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    |          
                    |--61.62%-- 0x2d6469646c697562
                    |          
                     --38.38%-- 0x6f2e7473696c7665

     1.48%     0.00%  ls       [unknown]           [.] 0x0909632e66666964               
                 |
                 ---0x909632e66666964

     1.44%     1.44%  ls       [kernel.vmlinux]    [k] paranoid_entry                   
                 |
                 ---paranoid_entry
                    __const_udelay
                    wait_for_xmitr
                    serial8250_console_putchar
                    uart_console_write
                    univ8250_console_write
                    call_console_drivers.constprop.27
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] perf_pmu_enable                  
                 |
                 ---perf_pmu_enable
                    |          
                    |--56.32%-- perf_event_context_sched_in
                    |          |          
                    |          |--100.00%-- __perf_event_task_sched_in
                    |          |          finish_task_switch
                    |          |          __schedule
                    |          |          preempt_schedule_common
                    |          |          _cond_resched
                    |          |          mutex_lock
                    |          |          n_tty_write
                    |          |          tty_write
                    |          |          __vfs_write
                    |          |          vfs_write
                    |          |          sys_write
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          __GI___libc_write
                    |          |          0x9096f2e61746164
                    |           --0.00%-- [...]
                    |          
                     --43.68%-- perf_pmu_sched_task
                               __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] x86_pmu_enable                   
                 |
                 ---x86_pmu_enable
                    perf_pmu_enable
                    |          
                    |--56.32%-- perf_event_context_sched_in
                    |          |          
                    |          |--100.00%-- __perf_event_task_sched_in
                    |          |          finish_task_switch
                    |          |          __schedule
                    |          |          preempt_schedule_common
                    |          |          _cond_resched
                    |          |          mutex_lock
                    |          |          n_tty_write
                    |          |          tty_write
                    |          |          __vfs_write
                    |          |          vfs_write
                    |          |          sys_write
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          __GI___libc_write
                    |          |          0x9096f2e61746164
                    |           --0.00%-- [...]
                    |          
                     --43.68%-- perf_pmu_sched_task
                               __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] intel_pmu_enable_all             
                 |
                 ---intel_pmu_enable_all
                    x86_pmu_enable
                    perf_pmu_enable
                    |          
                    |--56.32%-- perf_event_context_sched_in
                    |          |          
                    |          |--100.00%-- __perf_event_task_sched_in
                    |          |          finish_task_switch
                    |          |          __schedule
                    |          |          preempt_schedule_common
                    |          |          _cond_resched
                    |          |          mutex_lock
                    |          |          n_tty_write
                    |          |          tty_write
                    |          |          __vfs_write
                    |          |          vfs_write
                    |          |          sys_write
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          __GI___libc_write
                    |          |          0x9096f2e61746164
                    |           --0.00%-- [...]
                    |          
                     --43.68%-- perf_pmu_sched_task
                               __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] __intel_pmu_enable_all.isra.9    
                 |
                 ---__intel_pmu_enable_all.isra.9
                    intel_pmu_enable_all
                    x86_pmu_enable
                    perf_pmu_enable
                    |          
                    |--56.32%-- perf_event_context_sched_in
                    |          |          
                    |          |--100.00%-- __perf_event_task_sched_in
                    |          |          finish_task_switch
                    |          |          __schedule
                    |          |          preempt_schedule_common
                    |          |          _cond_resched
                    |          |          mutex_lock
                    |          |          n_tty_write
                    |          |          tty_write
                    |          |          __vfs_write
                    |          |          vfs_write
                    |          |          sys_write
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          __GI___libc_write
                    |          |          0x9096f2e61746164
                    |           --0.00%-- [...]
                    |          
                     --43.68%-- perf_pmu_sched_task
                               __perf_event_task_sched_out
                               __schedule
                               preempt_schedule_common
                               _cond_resched
                               mutex_lock
                               n_tty_write
                               tty_write
                               __vfs_write
                               vfs_write
                               sys_write
                               entry_SYSCALL_64_fastpath
                               __GI___libc_write
                               0x1dbdc70

     1.42%     0.00%  ls       [kernel.vmlinux]    [k] wq_worker_waking_up              
                 |
                 ---wq_worker_waking_up
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x657461746f6e6e61

     1.42%     1.42%  ls       [kernel.vmlinux]    [k] kthread_data                     
                 |
                 ---kthread_data
                    wq_worker_waking_up
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    do_output_char
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x657461746f6e6e61

     1.31%     0.04%  ls       [kernel.vmlinux]    [k] do_nmi                           
                 |
                 ---do_nmi
                    |          
                    |--52.08%-- end_repeat_nmi
                    |          |          
                    |          |--46.71%-- vsnprintf
                    |          |          sprintf
                    |          |          print_time.part.12
                    |          |          print_prefix
                    |          |          msg_print_text
                    |          |          console_unlock
                    |          |          vprintk_emit
                    |          |          vprintk_default
                    |          |          printk
                    |          |          perf_duration_warn
                    |          |          irq_work_run_list
                    |          |          irq_work_run
                    |          |          smp_irq_work_interrupt
                    |          |          irq_work_interrupt
                    |          |          __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |          |--41.63%-- __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |           --11.66%-- scheduler_tick
                    |                     update_process_times
                    |                     tick_sched_handle.isra.17
                    |                     tick_sched_timer
                    |                     __hrtimer_run_queues
                    |                     hrtimer_interrupt
                    |                     local_apic_timer_interrupt
                    |                     smp_apic_timer_interrupt
                    |                     apic_timer_interrupt
                    |                     setup_new_exec
                    |                     load_elf_binary
                    |                     search_binary_handler
                    |                     do_execveat_common.isra.32
                    |                     sys_execve
                    |                     return_from_execve
                    |                     0x7f9abb498a77
                    |          
                     --47.92%-- nmi
                               do_lookup_x

     1.29%     0.00%  ls       [unknown]           [.] 0x0000632e74617473               
                 |
                 ---0x632e74617473

     1.29%     1.29%  ls       ls                  [.] 0x0000000000005a40               
                 |
                 ---0x5a40
                    0x6dcb
                    0
                    0x632e74617473

     1.29%     0.00%  ls       ls                  [.] 0xffffffffffc05a40               
                 |
                 ---0x5a40
                    0x6dcb
                    0
                    0x632e74617473

     1.27%     0.00%  ls       [kernel.vmlinux]    [k] default_do_nmi                   
                 |
                 ---default_do_nmi
                    do_nmi
                    |          
                    |--50.48%-- end_repeat_nmi
                    |          |          
                    |          |--49.78%-- vsnprintf
                    |          |          sprintf
                    |          |          print_time.part.12
                    |          |          print_prefix
                    |          |          msg_print_text
                    |          |          console_unlock
                    |          |          vprintk_emit
                    |          |          vprintk_default
                    |          |          printk
                    |          |          perf_duration_warn
                    |          |          irq_work_run_list
                    |          |          irq_work_run
                    |          |          smp_irq_work_interrupt
                    |          |          irq_work_interrupt
                    |          |          __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |          |--44.37%-- __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |           --5.85%-- scheduler_tick
                    |                     update_process_times
                    |                     tick_sched_handle.isra.17
                    |                     tick_sched_timer
                    |                     __hrtimer_run_queues
                    |                     hrtimer_interrupt
                    |                     local_apic_timer_interrupt
                    |                     smp_apic_timer_interrupt
                    |                     apic_timer_interrupt
                    |                     setup_new_exec
                    |                     load_elf_binary
                    |                     search_binary_handler
                    |                     do_execveat_common.isra.32
                    |                     sys_execve
                    |                     return_from_execve
                    |                     0x7f9abb498a77
                    |          
                     --49.52%-- nmi
                               do_lookup_x

     1.27%     0.63%  ls       [kernel.vmlinux]    [k] nmi_handle                       
                 |
                 ---nmi_handle
                    default_do_nmi
                    do_nmi
                    |          
                    |--50.48%-- end_repeat_nmi
                    |          |          
                    |          |--49.78%-- vsnprintf
                    |          |          sprintf
                    |          |          print_time.part.12
                    |          |          print_prefix
                    |          |          msg_print_text
                    |          |          console_unlock
                    |          |          vprintk_emit
                    |          |          vprintk_default
                    |          |          printk
                    |          |          perf_duration_warn
                    |          |          irq_work_run_list
                    |          |          irq_work_run
                    |          |          smp_irq_work_interrupt
                    |          |          irq_work_interrupt
                    |          |          __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |          |--44.37%-- __inode_permission
                    |          |          inode_permission
                    |          |          link_path_walk
                    |          |          path_openat
                    |          |          do_filp_open
                    |          |          do_sys_open
                    |          |          sys_open
                    |          |          entry_SYSCALL_64_fastpath
                    |          |          open64
                    |          |          0x686361632e6f732e
                    |          |          
                    |           --5.85%-- scheduler_tick
                    |                     update_process_times
                    |                     tick_sched_handle.isra.17
                    |                     tick_sched_timer
                    |                     __hrtimer_run_queues
                    |                     hrtimer_interrupt
                    |                     local_apic_timer_interrupt
                    |                     smp_apic_timer_interrupt
                    |                     apic_timer_interrupt
                    |                     setup_new_exec
                    |                     load_elf_binary
                    |                     search_binary_handler
                    |                     do_execveat_common.isra.32
                    |                     sys_execve
                    |                     return_from_execve
                    |                     0x7f9abb498a77
                    |          
                     --49.52%-- nmi
                               do_lookup_x

     1.24%     0.00%  ls       [unknown]           [.] 0x0000000001dbdc70               
                 |
                 ---0x1dbdc70

     1.23%     0.00%  ls       [kernel.vmlinux]    [k] task_work_run                    
                 |
                 ---task_work_run
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.23%     0.00%  ls       [kernel.vmlinux]    [k] ____fput                         
                 |
                 ---____fput
                    task_work_run
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.23%     0.00%  ls       [kernel.vmlinux]    [k] __fput                           
                 |
                 ---__fput
                    ____fput
                    task_work_run
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.23%     1.23%  ls       [kernel.vmlinux]    [k] locks_remove_file                
                 |
                 ---locks_remove_file
                    __fput
                    ____fput
                    task_work_run
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.18%     0.00%  ls       [unknown]           [.] 0x096f2e68636e6562               
                 |
                 ---0x96f2e68636e6562

     1.10%     0.00%  ls       [kernel.vmlinux]    [k] tlb_finish_mmu                   
                 |
                 ---tlb_finish_mmu
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.10%     0.00%  ls       [kernel.vmlinux]    [k] tlb_flush_mmu_free               
                 |
                 ---tlb_flush_mmu_free
                    tlb_finish_mmu
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.10%     0.00%  ls       [kernel.vmlinux]    [k] free_pages_and_swap_cache        
                 |
                 ---free_pages_and_swap_cache
                    tlb_flush_mmu_free
                    tlb_finish_mmu
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.10%     1.10%  ls       [kernel.vmlinux]    [k] release_pages                    
                 |
                 ---release_pages
                    free_pages_and_swap_cache
                    tlb_flush_mmu_free
                    tlb_finish_mmu
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     1.08%     1.08%  ls       ls                  [.] 0x0000000000006b20               
                 |
                 ---0x6b20
                    0
                    0

     1.08%     0.00%  ls       ls                  [.] 0xffffffffffc06b20               
                 |
                 ---0x6b20
                    0
                    0

     1.01%     0.00%  ls       [unknown]           [.] 0x72616863656d6974               
                 |
                 ---0x72616863656d6974

     1.01%     1.01%  ls       ls                  [.] __ctype_get_mb_cur_max@plt       
                 |
                 ---__ctype_get_mb_cur_max@plt
                    0x6dcb
                    0
                    0x72616863656d6974

     1.01%     1.01%  ls       [kernel.vmlinux]    [k] unmap_page_range                 
                 |
                 ---unmap_page_range
                    unmap_single_vma
                    unmap_vmas
                    exit_mmap
                    mmput
                    do_exit
                    do_group_exit
                    0x27cb44
                    entry_SYSCALL_64_fastpath

     0.94%     0.00%  ls       [unknown]           [.] 0x2d6469646c697562               
                 |
                 ---0x2d6469646c697562

     0.89%     0.89%  ls       [kernel.vmlinux]    [k] get_pwq.isra.18                  
                 |
                 ---get_pwq.isra.18
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write

     0.80%     0.00%  ls       [kernel.vmlinux]    [k] end_repeat_nmi                   
                 |
                 ---end_repeat_nmi
                    |          
                    |--39.76%-- vsnprintf
                    |          sprintf
                    |          print_time.part.12
                    |          print_prefix
                    |          msg_print_text
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                    |--35.44%-- __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --24.80%-- scheduler_tick
                               update_process_times
                               tick_sched_handle.isra.17
                               tick_sched_timer
                               __hrtimer_run_queues
                               hrtimer_interrupt
                               local_apic_timer_interrupt
                               smp_apic_timer_interrupt
                               apic_timer_interrupt
                               setup_new_exec
                               load_elf_binary
                               search_binary_handler
                               do_execveat_common.isra.32
                               sys_execve
                               return_from_execve
                               0x7f9abb498a77

     0.80%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_context_sched_in      
                 |
                 ---perf_event_context_sched_in
                    |          
                    |--100.00%-- __perf_event_task_sched_in
                    |          finish_task_switch
                    |          __schedule
                    |          preempt_schedule_common
                    |          _cond_resched
                    |          mutex_lock
                    |          n_tty_write
                    |          tty_write
                    |          __vfs_write
                    |          vfs_write
                    |          sys_write
                    |          entry_SYSCALL_64_fastpath
                    |          __GI___libc_write
                    |          0x9096f2e61746164
                     --0.00%-- [...]

     0.80%     0.00%  ls       [unknown]           [.] 0x09096f2e61746164               
                 |
                 ---0x9096f2e61746164

     0.78%     0.00%  ls       libc-2.18.so        [.] __run_exit_handlers              
                 |
                 ---__run_exit_handlers
                    0

     0.78%     0.00%  ls       libc-2.18.so        [.] _exit                            
                 |
                 ---_exit
                    __run_exit_handlers
                    0

     0.72%     0.72%  ls       [kernel.vmlinux]    [k] account_entity_enqueue           
                 |
                 ---account_entity_enqueue
                    enqueue_entity
                    enqueue_task_fair
                    enqueue_task
                    activate_task
                    ttwu_do_activate.constprop.92
                    try_to_wake_up
                    wake_up_process
                    wake_up_worker
                    insert_work
                    __queue_work
                    queue_work_on
                    tty_schedule_flip
                    tty_flip_buffer_push
                    pty_write
                    n_tty_write
                    tty_write
                    __vfs_write
                    vfs_write
                    sys_write
                    entry_SYSCALL_64_fastpath
                    __GI___libc_write
                    0x909632e66666964

     0.62%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_nmi_handler           
                 |
                 ---perf_event_nmi_handler
                    nmi_handle
                    default_do_nmi
                    do_nmi
                    end_repeat_nmi
                    |          
                    |--51.70%-- vsnprintf
                    |          sprintf
                    |          print_time.part.12
                    |          print_prefix
                    |          msg_print_text
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                    |--46.08%-- __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --2.22%-- scheduler_tick
                               update_process_times
                               tick_sched_handle.isra.17
                               tick_sched_timer
                               __hrtimer_run_queues
                               hrtimer_interrupt
                               local_apic_timer_interrupt
                               smp_apic_timer_interrupt
                               apic_timer_interrupt
                               setup_new_exec
                               load_elf_binary
                               search_binary_handler
                               do_execveat_common.isra.32
                               sys_execve
                               return_from_execve
                               0x7f9abb498a77

     0.60%     0.60%  ls       [kernel.vmlinux]    [k] perf_sample_event_took           
                 |
                 ---perf_sample_event_took
                    perf_event_nmi_handler
                    nmi_handle
                    default_do_nmi
                    do_nmi
                    end_repeat_nmi
                    |          
                    |--52.87%-- vsnprintf
                    |          sprintf
                    |          print_time.part.12
                    |          print_prefix
                    |          msg_print_text
                    |          console_unlock
                    |          vprintk_emit
                    |          vprintk_default
                    |          printk
                    |          perf_duration_warn
                    |          irq_work_run_list
                    |          irq_work_run
                    |          smp_irq_work_interrupt
                    |          irq_work_interrupt
                    |          __inode_permission
                    |          inode_permission
                    |          link_path_walk
                    |          path_openat
                    |          do_filp_open
                    |          do_sys_open
                    |          sys_open
                    |          entry_SYSCALL_64_fastpath
                    |          open64
                    |          0x686361632e6f732e
                    |          
                     --47.13%-- __inode_permission
                               inode_permission
                               link_path_walk
                               path_openat
                               do_filp_open
                               do_sys_open
                               sys_open
                               entry_SYSCALL_64_fastpath
                               open64
                               0x686361632e6f732e

     0.58%     0.00%  ls       [unknown]           [.] 0x6f2e7473696c7665               
                 |
                 ---0x6f2e7473696c7665

     0.58%     0.00%  ls       [kernel.vmlinux]    [k] pick_next_task_fair              
                 |
                 ---pick_next_task_fair
                    __schedule
                    schedule
                    prepare_exit_to_usermode
                    syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    0x6f2e7473696c7665

     0.58%     0.58%  ls       [kernel.vmlinux]    [k] __enqueue_entity                 
                 |
                 ---__enqueue_entity
                    pick_next_task_fair
                    __schedule
                    schedule
                    prepare_exit_to_usermode
                    syscall_return_slowpath
                    int_ret_from_sys_call
                    __GI___libc_write
                    0x6f2e7473696c7665

     0.57%     0.00%  ls       libc-2.18.so        [.] __GI___munmap                    
                 |
                 ---__GI___munmap

     0.57%     0.00%  ls       [kernel.vmlinux]    [k] sys_munmap                       
                 |
                 ---sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.57%     0.00%  ls       [kernel.vmlinux]    [k] vm_munmap                        
                 |
                 ---vm_munmap
                    sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.57%     0.00%  ls       [kernel.vmlinux]    [k] do_munmap                        
                 |
                 ---do_munmap
                    vm_munmap
                    sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.57%     0.00%  ls       [kernel.vmlinux]    [k] remove_vma                       
                 |
                 ---remove_vma
                    do_munmap
                    vm_munmap
                    sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.57%     0.57%  ls       [kernel.vmlinux]    [k] kmem_cache_free                  
                 |
                 ---kmem_cache_free
                    remove_vma
                    do_munmap
                    vm_munmap
                    sys_munmap
                    entry_SYSCALL_64_fastpath
                    __GI___munmap

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] msg_print_text                   
                 |
                 ---msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] print_prefix                     
                 |
                 ---print_prefix
                    msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] print_time.part.12               
                 |
                 ---print_time.part.12
                    print_prefix
                    msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] sprintf                          
                 |
                 ---sprintf
                    print_time.part.12
                    print_prefix
                    msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       [kernel.vmlinux]    [k] vsnprintf                        
                 |
                 ---vsnprintf
                    sprintf
                    print_time.part.12
                    print_prefix
                    msg_print_text
                    console_unlock
                    vprintk_emit
                    vprintk_default
                    printk
                    perf_duration_warn
                    irq_work_run_list
                    irq_work_run
                    smp_irq_work_interrupt
                    irq_work_interrupt
                    __inode_permission
                    inode_permission
                    link_path_walk
                    path_openat
                    do_filp_open
                    do_sys_open
                    sys_open
                    entry_SYSCALL_64_fastpath
                    open64
                    0x686361632e6f732e

     0.32%     0.00%  ls       ld-2.18.so          [.] 0xffff804cabb62640               
                 |
                 ---0x21e640
                    0

     0.32%     0.32%  ls       ld-2.18.so          [.] _dl_setup_hash                   
                 |
                 ---_dl_setup_hash
                    _dl_map_object
                    0x21e640
                    0

     0.29%     0.00%  ls       [unknown]           [k] 0x00007f9abb498a77               
                 |
                 ---0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] return_from_execve               
                 |
                 ---return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] sys_execve                       
                 |
                 ---sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] do_execveat_common.isra.32       
                 |
                 ---do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] search_binary_handler            
                 |
                 ---search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] load_elf_binary                  
                 |
                 ---load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] setup_new_exec                   
                 |
                 ---setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] apic_timer_interrupt             
                 |
                 ---apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] smp_apic_timer_interrupt         
                 |
                 ---smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] local_apic_timer_interrupt       
                 |
                 ---local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] hrtimer_interrupt                
                 |
                 ---hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] __hrtimer_run_queues             
                 |
                 ---__hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] tick_sched_timer                 
                 |
                 ---tick_sched_timer
                    __hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] tick_sched_handle.isra.17        
                 |
                 ---tick_sched_handle.isra.17
                    tick_sched_timer
                    __hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] update_process_times             
                 |
                 ---update_process_times
                    tick_sched_handle.isra.17
                    tick_sched_timer
                    __hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.21%     0.00%  ls       [kernel.vmlinux]    [k] scheduler_tick                   
                 |
                 ---scheduler_tick
                    update_process_times
                    tick_sched_handle.isra.17
                    tick_sched_timer
                    __hrtimer_run_queues
                    hrtimer_interrupt
                    local_apic_timer_interrupt
                    smp_apic_timer_interrupt
                    apic_timer_interrupt
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77

     0.08%     0.08%  ls       [kernel.vmlinux]    [k] native_sched_clock               
                 |
                 ---native_sched_clock
                    0x7f9abb498a77

     0.00%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_exec                  
                 |
                 ---perf_event_exec
                    setup_new_exec
                    load_elf_binary
                    search_binary_handler
                    do_execveat_common.isra.32
                    sys_execve
                    return_from_execve
                    0x7f9abb498a77



# Samples: 158  of event 'cpu/instructions,call-graph=no,time=0,period=20000/p'
# Event count (approx.): 3762761
#
# Children      Self  Command  Shared Object     Symbol                            
# ........  ........  .......  ................  ..................................
#
     5.87%     5.87%  ls       [unknown]         [.] 0x00007fb353d5cf7e            

     4.84%     4.84%  ls       [kernel.vmlinux]  [k] delay_tsc                     

     3.77%     3.77%  ls       [kernel.vmlinux]  [k] pick_next_task_fair           

     3.11%     3.11%  ls       [kernel.vmlinux]  [k] unmap_page_range              

     2.94%     2.94%  ls       [kernel.vmlinux]  [k] n_tty_write                   

     2.48%     2.48%  ls       [unknown]         [.] 0x000000000040669a            

     2.40%     2.40%  ls       [unknown]         [.] 0x00007fb353d5cd1a            

     2.37%     2.37%  ls       [unknown]         [.] 0x00007fb353d5cca4            

     2.37%     2.37%  ls       [unknown]         [.] 0x00000000004066ca            

     2.25%     2.25%  ls       [unknown]         [.] 0x00007fb353d5db87            

     2.03%     2.03%  ls       [unknown]         [.] 0x00007fb353d5ccaa            

     1.98%     1.98%  ls       [unknown]         [.] 0x00007fb353d5cc9c            

     1.97%     1.97%  ls       [unknown]         [.] 0x0000000000405b2a            

     1.89%     1.89%  ls       [unknown]         [.] 0x00007fb3546d43e0            

     1.83%     1.83%  ls       [unknown]         [.] 0x00007fb353d5cef6            

     1.75%     1.75%  ls       [kernel.vmlinux]  [k] ring_buffer_record_is_on      

     1.69%     1.69%  ls       [unknown]         [.] 0x00007fb353d5cc3a            

     1.63%     1.63%  ls       [unknown]         [.] 0x0000000000405611            

     1.47%     1.47%  ls       [kernel.vmlinux]  [k] __fput                        

     1.44%     1.44%  ls       [kernel.vmlinux]  [k] release_pages                 

     1.37%     1.37%  ls       [kernel.vmlinux]  [k] perf_pmu_sched_task           

     1.34%     1.34%  ls       [unknown]         [.] 0x00007fb353d54fed            

     1.33%     1.33%  ls       [kernel.vmlinux]  [k] __memcpy                      

     1.33%     1.33%  ls       [unknown]         [.] 0x00007fb353d5df34            

     1.31%     1.31%  ls       [unknown]         [.] 0x00007fb353d5ccdf            

     1.29%     1.29%  ls       [unknown]         [.] 0x0000000000406a19            

     1.29%     1.29%  ls       [kernel.vmlinux]  [k] page_remove_rmap              

     1.29%     1.29%  ls       [kernel.vmlinux]  [k] do_set_pte                    

     1.28%     1.28%  ls       [kernel.vmlinux]  [k] __audit_syscall_exit          

     1.28%     1.28%  ls       [kernel.vmlinux]  [k] enqueue_entity                

     1.28%     1.28%  ls       [ext4]            [k] ext4fs_dirhash                

     1.26%     1.26%  ls       [kernel.vmlinux]  [k] cpuacct_charge                

     1.24%     1.24%  ls       [kernel.vmlinux]  [k] sched_clock                   

     1.24%     1.24%  ls       [unknown]         [.] 0x00007fb353d4bca5            

     1.22%     1.22%  ls       [unknown]         [.] 0x00007fb353dfb6ac            

     1.21%     1.21%  ls       [ext4]            [k] str2hashbuf_signed            

     1.21%     1.21%  ls       [unknown]         [.] 0x00007fb3546c766a            

     1.20%     1.20%  ls       [ext4]            [k] __ext4_check_dir_entry        

     1.13%     1.13%  ls       [kernel.vmlinux]  [k] x86_pmu_disable               

     1.12%     1.12%  ls       [unknown]         [.] 0x00007fb3546c5571            

     1.11%     1.11%  ls       [kernel.vmlinux]  [k] vm_normal_page                

     1.09%     1.09%  ls       [unknown]         [.] 0x00007fb3546cc246            

     1.06%     1.06%  ls       [unknown]         [.] 0x00007fb3546cc21e            

     1.03%     1.03%  ls       [unknown]         [.] 0x000000000040e6a0            

     1.01%     1.01%  ls       [unknown]         [.] 0x00007fb3546c5ce3            

     0.88%     0.88%  ls       [unknown]         [.] 0x00007fb3546c7814            

     0.88%     0.88%  ls       [kernel.vmlinux]  [k] mark_page_accessed            

     0.87%     0.87%  ls       [unknown]         [.] 0x00007fb3546c7960            

     0.78%     0.78%  ls       [unknown]         [.] 0x000000000040e7ea            

     0.77%     0.77%  ls       [kernel.vmlinux]  [k] security_mmap_file            

     0.66%     0.66%  ls       [unknown]         [.] 0x000000000040f3c5            

     0.65%     0.65%  ls       [kernel.vmlinux]  [k] enqueue_task_fair             

     0.63%     0.63%  ls       [kernel.vmlinux]  [k] unlock_page                   

     0.61%     0.61%  ls       [kernel.vmlinux]  [k] place_entity                  

     0.57%     0.57%  ls       [unknown]         [.] 0x000000000040e808            

     0.56%     0.56%  ls       [kernel.vmlinux]  [k] perf_event_aux_ctx            

     0.54%     0.54%  ls       [kernel.vmlinux]  [k] __rb_insert_augmented         

     0.54%     0.54%  ls       [kernel.vmlinux]  [k] update_min_vruntime           

     0.50%     0.50%  ls       [kernel.vmlinux]  [k] update_curr                   

     0.47%     0.47%  ls       [kernel.vmlinux]  [k] vma_interval_tree_insert      

     0.47%     0.47%  ls       [kernel.vmlinux]  [k] io_serial_in                  

     0.47%     0.47%  ls       [kernel.vmlinux]  [k] check_cfs_rq_runtime          

     0.46%     0.46%  ls       [kernel.vmlinux]  [k] tracing_record_cmdline        

     0.46%     0.46%  ls       [kernel.vmlinux]  [k] __dec_zone_page_state         

     0.42%     0.42%  ls       [kernel.vmlinux]  [k] try_to_wake_up                

     0.40%     0.40%  ls       [kernel.vmlinux]  [k] insert_work                   

     0.40%     0.40%  ls       [kernel.vmlinux]  [k] put_prev_entity               

     0.40%     0.40%  ls       [kernel.vmlinux]  [k] avc_has_perm_noaudit          

     0.38%     0.38%  ls       [kernel.vmlinux]  [k] __list_del_entry              

     0.38%     0.38%  ls       [unknown]         [.] 0x0000000000405852            

     0.36%     0.36%  ls       [kernel.vmlinux]  [k] __intel_pmu_disable_all       

     0.35%     0.35%  ls       [kernel.vmlinux]  [k] ttwu_do_activate.constprop.92 

     0.35%     0.35%  ls       [kernel.vmlinux]  [k] mem_cgroup_begin_page_stat    

     0.34%     0.34%  ls       [kernel.vmlinux]  [k] perf_ctx_unlock               

     0.32%     0.32%  ls       [unknown]         [.] 0x0000000000405b46            

     0.32%     0.32%  ls       [unknown]         [.] 0x0000000000405811            

     0.31%     0.31%  ls       [unknown]         [.] 0x00007fb3546c54fa            

     0.31%     0.31%  ls       [kernel.vmlinux]  [k] rb_next                       

     0.31%     0.31%  ls       [kernel.vmlinux]  [k] tracing_is_on                 

     0.30%     0.30%  ls       [unknown]         [.] 0x00007fb3546c553d            

     0.29%     0.29%  ls       [unknown]         [.] 0x00007fb3546cb597            

     0.20%     0.20%  ls       [kernel.vmlinux]  [k] format_decode                 

     0.18%     0.18%  ls       [kernel.vmlinux]  [k] __audit_syscall_entry         

     0.16%     0.16%  ls       [kernel.vmlinux]  [k] arch_get_unmapped_area_topdown

     0.16%     0.16%  ls       [kernel.vmlinux]  [k] fsnotify                      

     0.16%     0.16%  ls       [kernel.vmlinux]  [k] intel_bts_enable_local        

     0.15%     0.15%  ls       [kernel.vmlinux]  [k] selinux_inode_permission      

     0.14%     0.14%  ls       [kernel.vmlinux]  [k] wait_for_xmitr                

     0.14%     0.14%  ls       [kernel.vmlinux]  [k] __const_udelay                

     0.11%     0.11%  ls       [kernel.vmlinux]  [k] find_vma                      

     0.11%     0.11%  ls       [unknown]         [.] 0x00007fb3546d18ae            

     0.10%     0.10%  ls       [unknown]         [.] 0x00007fb3546cb18e            

     0.03%     0.03%  ls       [kernel.vmlinux]  [k] native_sched_clock            

     0.01%     0.01%  ls       [kernel.vmlinux]  [k] trigger_load_balance          

     0.01%     0.01%  ls       [kernel.vmlinux]  [k] do_nmi                        

     0.00%     0.00%  ls       [kernel.vmlinux]  [k] perf_event_nmi_handler        

     0.00%     0.00%  ls       [kernel.vmlinux]  [k] perf_sample_event_took        

     0.00%     0.00%  ls       [kernel.vmlinux]  [k] nmi                           



#
# (For a higher level overview, try: perf report --sort comm,dso)
#

^ permalink raw reply	[flat|nested] 20+ messages in thread

* [PATCH] perf tools: Unset perf_event_attr::freq when period term is set
  2015-08-07 15:38   ` Arnaldo Carvalho de Melo
  2015-08-07 15:49     ` Arnaldo Carvalho de Melo
@ 2015-08-08 17:12     ` Jiri Olsa
  2015-08-12 12:29       ` [tip:perf/core] perf tools: Unset perf_event_attr:: freq " tip-bot for Jiri Olsa
  1 sibling, 1 reply; 20+ messages in thread
From: Jiri Olsa @ 2015-08-08 17:12 UTC (permalink / raw)
  To: Arnaldo Carvalho de Melo; +Cc: kan.liang, jolsa, namhyung, ak, linux-kernel

On Fri, Aug 07, 2015 at 12:38:43PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Thu, Aug 06, 2015 at 03:44:53PM -0400, kan.liang@intel.com escreveu:
> > From: Kan Liang <kan.liang@intel.com>
> > Here are some examples and test results.
> > 
> > 1. Comparing the elapsed time and perf.data size from "kernbench -M -H".
> > 
> >  The test command for FULL callgraph and time support.
> >    "perf record -e
> >    '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}'
> >    --call-graph fp --time"
> 
> Jiri, while testing this I noticed that the message for EINVAL when
> using the cpu// syntax (per-event settings) is cryptic:
> 
>   [root@zoo ~]# perf record -e 'cpu/cpu-cycles,call-graph=fp,time,period=100000/p' ls
>   Error:
>   The sys_perf_event_open() syscall returned with 22 (Invalid argument) for event (cpu/cpu-cycles,call-graph=fp,time,period=100000/p).
>   /bin/dmesg may provide additional information.
>   No CONFIG_PERF_EVENTS=y kernel support configured?
> 
> Whereas if we use -F, it is much, much clearer, telling the user exactly
> what is failing and what needs to be done to make it work:
> 
>   [root@zoo ~]# perf record -F 100000 -e cpu/cpu-cycles/ usleep 1
>   Maximum frequency rate (25000) reached.
>   Please use -F freq option with lower value or consider
>   tweaking /proc/sys/kernel/perf_event_max_sample_rate.
>   [root@zoo ~]# 
> 
> Hope this is something easy to wire up, given your event parsing kung foo
> skillz...

my kungu foo found there was actually another issue ;-)

we did not clear up attr->freq bit.. so the sample_period
was handled as sample_freq value.. please check the patch
below

anyfoo, now period setup can raise sky high (which is correct)
and once it meets the God it shows:

[jolsa@krava perf]$ ./perf record -e 'cpu/cpu-cycles,call-graph=fp,time,period=1000000000000000000000000/' ls
event syntax error: '..time,period=1000000000000000000000000/'
                                  \___ parser error

I'll check if we could add some default error message when value
crosses the type maximum.. which now falls to parser error

jirka


---
We need to unset 'perf_event_attr::freq' bit (default 1) when
'period' term is specified within event definition like:

  -e 'cpu/cpu-cycles,call-graph=fp,time,period=100000'

otherwise it will handle the period value as frequency
(and fail if it crossed the maximum allowed frequency value).

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Link: http://lkml.kernel.org/n/tip-7f3gjwbekakhyxr47wvin4zb@git.kernel.org
---
diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c
index d902cad4ce10..4c779d23b1d7 100644
--- a/tools/perf/util/evsel.c
+++ b/tools/perf/util/evsel.c
@@ -623,6 +623,7 @@ static void apply_config_terms(struct perf_evsel *evsel,
 		switch (term->type) {
 		case PERF_EVSEL__CONFIG_TERM_PERIOD:
 			attr->sample_period = term->val.period;
+			attr->freq = 0;
 			break;
 		case PERF_EVSEL__CONFIG_TERM_TIME:
 			if (term->val.time)

^ permalink raw reply related	[flat|nested] 20+ messages in thread

* Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support
  2015-08-08 16:45       ` Jiri Olsa
@ 2015-08-08 17:35         ` Arnaldo Carvalho de Melo
  2015-08-10 13:46         ` Arnaldo Carvalho de Melo
  1 sibling, 0 replies; 20+ messages in thread
From: Arnaldo Carvalho de Melo @ 2015-08-08 17:35 UTC (permalink / raw)
  To: Jiri Olsa; +Cc: Kan Liang, Jiri Olsa, Namhyung Kim, Andi Kleen, linux-kernel

Em Sat, Aug 08, 2015 at 06:45:39PM +0200, Jiri Olsa escreveu:
> On Fri, Aug 07, 2015 at 12:49:38PM -0300, Arnaldo Carvalho de Melo wrote:
> > Em Fri, Aug 07, 2015 at 12:38:43PM -0300, Arnaldo Carvalho de Melo escreveu:
> > > Em Thu, Aug 06, 2015 at 03:44:53PM -0400, kan.liang@intel.com escreveu:
> > > > From: Kan Liang <kan.liang@intel.com>
> > > > Here are some examples and test results.
> > > > 
> > > > 1. Comparing the elapsed time and perf.data size from "kernbench -M -H".
> > > > 
> > > >  The test command for FULL callgraph and time support.
> > > >    "perf record -e
> > > >    '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}'
> > > >    --call-graph fp --time"
> > 
> > Some other oddity I noticed while testing:
> > 
> 
> SNIP
> 
> > This is in the 'perf report' TUI, why, for an event with 'callgraph=no', we
> > get callchains? How come?
> > 
> > If I try it with the --stdio, well, its there as well:
> > 
> > # Samples: 2K of event 'cpu/instructions,call-graph=no,time=0,period=20000/p'
> > # Event count (approx.): 46956518
> > #
> > # Children      Self  Command          Shared Object               Symbol                                        
> > # ........  ........  ...............  ..........................  ..............................................
> > #
> >     67.56%     0.00%  qemu-system-x86  [unknown]                   [.] 0xad5e258d4c544155                        
> >             |
> >             ---0xad5e258d4c544155
> > 
> >     67.56%     0.00%  qemu-system-x86  libc-2.20.so                [.] __libc_start_main                         
> >             |
> >             ---__libc_start_main
> >                0xad5e258d4c544155
> > 
> >     67.56%     0.00%  qemu-system-x86  perf                        [.] main                                      
> >             |
> >             ---main
> >                __libc_start_main
> >                0xad5e258d4c544155
> > 
> >     67.56%     0.00%  qemu-system-x86  perf                        [.] run_builtin                               
> >             |
> >             ---run_builtin
> >                main
> >                __libc_start_main
> >                0xad5e258d4c544155
> > 
> > Can you take a look? What I have, i.e. the patches from Jiri and Kan
> > moving stuff out of the way of the python binding and this patch is at
> > the tmp.perf/core branch in my git tree:
> > 
> >   git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git
> 
> hum, tried this branch and got proper behaviour for both stdio and TUI,
> attaching stdio output
> 
> the record command was:
>   $ ./perf record -e 'cpu/cpu-cycles,call-graph=fp,time,period=10000/pp,cpu/instructions,call-graph=no,time=0,period=20000/p' ls
> 
> got the events strings from your report output.. what was different in your record command?

Was in my message, here it is:

# cmdline : /home/acme/bin/perf record -e {cpu/cpu-cycles,call-graph=fp,time,period=10000/pp,cpu/instructions,call-graph=no,time=0,period=20000/p}
-a

I'll try this again later.

- Arnaldo

^ permalink raw reply	[flat|nested] 20+ messages in thread

* RE: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support
  2015-08-07 15:49     ` Arnaldo Carvalho de Melo
  2015-08-08 16:45       ` Jiri Olsa
@ 2015-08-10 12:56       ` Liang, Kan
  2015-08-10 15:39         ` Arnaldo Carvalho de Melo
  1 sibling, 1 reply; 20+ messages in thread
From: Liang, Kan @ 2015-08-10 12:56 UTC (permalink / raw)
  To: Arnaldo Carvalho de Melo
  Cc: Jiri Olsa, Namhyung Kim, Andi Kleen, linux-kernel

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset="utf-8", Size: 6150 bytes --]



> Em Fri, Aug 07, 2015 at 12:38:43PM -0300, Arnaldo Carvalho de Melo
> escreveu:
> > Em Thu, Aug 06, 2015 at 03:44:53PM -0400, kan.liang@intel.com escreveu:
> > > From: Kan Liang <kan.liang@intel.com> Here are some examples and
> > > test results.
> > >
> > > 1. Comparing the elapsed time and perf.data size from "kernbench -M -
> H".
> > >
> > >  The test command for FULL callgraph and time support.
> > >    "perf record -e
> > >    '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}'
> > >    --call-graph fp --time"
> 
> Some other oddity I noticed while testing:
> 
> For:
> 
> [root@zoo ~]# perf report --header-only
> # ========
> # captured on: Fri Aug  7 12:39:13 2015
> # hostname : zoo
> # os release : 4.2.0-rc5+
> # perf version : 4.2.rc4.g1cd951
> # arch : x86_64
> # nrcpus online : 4
> # nrcpus avail : 4
> # cpudesc : Intel(R) Core(TM) i7-3667U CPU @ 2.00GHz # cpuid :
> GenuineIntel,6,58,9 # total memory : 8080692 kB # cmdline :
> /home/acme/bin/perf record -e {cpu/cpu-cycles,call-
> graph=fp,time,period=10000/pp,cpu/instructions,call-
> graph=no,time=0,period=20000/p} -a # event : name = cpu/cpu-cycles,call-
> graph=fp,time,period=10000/pp, , id = { 38537, 38538, 38539, 38540 }, type
> = 4, size = 112, config = 0x3c, { sample_perio # event : name =
> cpu/instructions,call-graph=no,time=0,period=20000/p, , id = { 38541,
> 38542, 38543, 38544 }, type = 4, size = 112, config = 0xc0, { sample_pe #
> HEADER_CPU_TOPOLOGY info available, use -I to display #
> HEADER_NUMA_TOPOLOGY info available, use -I to display # pmu
> mappings: cpu = 4, software = 1, power = 7, uncore_imc = 8, tracepoint = 2,
> uncore_cbox_0 = 9, uncore_cbox_1 = 10, breakpoint = 5 # group:
> {cpu/cpu-cycles,call-graph=fp,time,period=10000/pp,cpu/instructions,call-
> graph=no,time=0,period=20000/p}
> # ========
> #
> 
> I get:
> 
> Samples: 2K of event 'cpu/instructions,call-
> graph=no,time=0,period=20000/p', Event count (approx.): 46956518
>   Children      Self  Command          Shared Object Symbol                  â—†
> -   67.56%     0.00%  qemu-system-x86  [unknown]     [.]
> 0xad5e258d4c544155  â–’
>      0xad5e258d4c544155                                                      â–’
> -   67.56%     0.00%  qemu-system-x86  libc-2.20.so  [.] __libc_start_main   â–’
>      __libc_start_main                                                       â–’
>      0xad5e258d4c544155                                                      â–’
> -   67.56%     0.00%  qemu-system-x86  perf          [.] main                â–’
>      main                                                                    â–’
>      __libc_start_main                                                       â–’
>      0xad5e258d4c544155                                                      â–’
> -   67.56%     0.00%  qemu-system-x86  perf          [.] run_builtin         â–’
>      run_builtin                                                             â–’
>      main                                                                    â–’
>      __libc_start_main                                                       â–’
> 
> This is in the 'perf report' TUI, why, for an event with 'callgraph=no', we get
> callchains? How come?
> 

That's the design.
For sampling multiple events, it may not be needed to collect callgraphs for
all of them. Because the sample sites are usually nearby. It's enough to collect
the callgraphs on a reference event. For other events, it can still show callgraphs
according to the callgraphs on a reference event.

Thanks,
Kan

> If I try it with the --stdio, well, its there as well:
> 
> # Samples: 2K of event 'cpu/instructions,call-
> graph=no,time=0,period=20000/p'
> # Event count (approx.): 46956518
> #
> # Children      Self  Command          Shared Object               Symbol
> # ........  ........  ...............  ..........................  ..............................................
> #
>     67.56%     0.00%  qemu-system-x86  [unknown]                   [.]
> 0xad5e258d4c544155
>             |
>             ---0xad5e258d4c544155
> 
>     67.56%     0.00%  qemu-system-x86  libc-2.20.so                [.]
> __libc_start_main
>             |
>             ---__libc_start_main
>                0xad5e258d4c544155
> 
>     67.56%     0.00%  qemu-system-x86  perf                        [.] main
>             |
>             ---main
>                __libc_start_main
>                0xad5e258d4c544155
> 
>     67.56%     0.00%  qemu-system-x86  perf                        [.] run_builtin
>             |
>             ---run_builtin
>                main
>                __libc_start_main
>                0xad5e258d4c544155
> 
> Can you take a look? What I have, i.e. the patches from Jiri and Kan moving
> stuff out of the way of the python binding and this patch is at the
> tmp.perf/core branch in my git tree:
> 
>   git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git
> 
> - Arnaldo
> 
> 
> > Jiri, while testing this I noticed that the message for EINVAL when
> > using the cpu// syntax (per-event settings) is cryptic:
> >
> >   [root@zoo ~]# perf record -e 'cpu/cpu-cycles,call-
> graph=fp,time,period=100000/p' ls
> >   Error:
> >   The sys_perf_event_open() syscall returned with 22 (Invalid argument)
> for event (cpu/cpu-cycles,call-graph=fp,time,period=100000/p).
> >   /bin/dmesg may provide additional information.
> >   No CONFIG_PERF_EVENTS=y kernel support configured?
> >
> > Whereas if we use -F, it is much, much clearer, telling the user
> > exactly what is failing and what needs to be done to make it work:
> >
> >   [root@zoo ~]# perf record -F 100000 -e cpu/cpu-cycles/ usleep 1
> >   Maximum frequency rate (25000) reached.
> >   Please use -F freq option with lower value or consider
> >   tweaking /proc/sys/kernel/perf_event_max_sample_rate.
> >   [root@zoo ~]#
> >
> > Hope this is something easy to wire up, given your event parsing kung
> > foo skillz...
> >
> > ;-)
> >
> > - Arnaldo
ÿôèº{.nÇ+‰·Ÿ®‰­†+%ŠËÿ±éݶ\x17¥Šwÿº{.nÇ+‰·¥Š{±þG«éÿŠ{ayº\x1dʇڙë,j\a­¢f£¢·hšïêÿ‘êçz_è®\x03(­éšŽŠÝ¢j"ú\x1a¶^[m§ÿÿ¾\a«þG«éÿ¢¸?™¨è­Ú&£ø§~á¶iO•æ¬z·švØ^\x14\x04\x1a¶^[m§ÿÿÃ\fÿ¶ìÿ¢¸?–I¥

^ permalink raw reply	[flat|nested] 20+ messages in thread

* Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support
  2015-08-08 16:45       ` Jiri Olsa
  2015-08-08 17:35         ` Arnaldo Carvalho de Melo
@ 2015-08-10 13:46         ` Arnaldo Carvalho de Melo
  1 sibling, 0 replies; 20+ messages in thread
From: Arnaldo Carvalho de Melo @ 2015-08-10 13:46 UTC (permalink / raw)
  To: Jiri Olsa; +Cc: Kan Liang, Jiri Olsa, Namhyung Kim, Andi Kleen, linux-kernel

Em Sat, Aug 08, 2015 at 06:45:39PM +0200, Jiri Olsa escreveu:
> On Fri, Aug 07, 2015 at 12:49:38PM -0300, Arnaldo Carvalho de Melo wrote:
> > Em Fri, Aug 07, 2015 at 12:38:43PM -0300, Arnaldo Carvalho de Melo escreveu:
> > > Em Thu, Aug 06, 2015 at 03:44:53PM -0400, kan.liang@intel.com escreveu:
> > > > From: Kan Liang <kan.liang@intel.com>
> > > > Here are some examples and test results.
> > > > 
> > > > 1. Comparing the elapsed time and perf.data size from "kernbench -M -H".
> > > > 
> > > >  The test command for FULL callgraph and time support.
> > > >    "perf record -e
> > > >    '{cpu/cpu-cycles,period=100000/,cpu/instructions,period=20000/p}'
> > > >    --call-graph fp --time"
> > 
> > Some other oddity I noticed while testing:
> > 
> 
> SNIP
> 
> > This is in the 'perf report' TUI, why, for an event with 'callgraph=no', we
> > get callchains? How come?
> > 
> > If I try it with the --stdio, well, its there as well:
> > 
> > # Samples: 2K of event 'cpu/instructions,call-graph=no,time=0,period=20000/p'
> > # Event count (approx.): 46956518
> > #
> > # Children      Self  Command          Shared Object               Symbol                                        
> > # ........  ........  ...............  ..........................  ..............................................
> > #
> >     67.56%     0.00%  qemu-system-x86  [unknown]                   [.] 0xad5e258d4c544155                        
> >             |
> >             ---0xad5e258d4c544155
> > 
> >     67.56%     0.00%  qemu-system-x86  libc-2.20.so                [.] __libc_start_main                         
> >             |
> >             ---__libc_start_main
> >                0xad5e258d4c544155
> > 
> >     67.56%     0.00%  qemu-system-x86  perf                        [.] main                                      
> >             |
> >             ---main
> >                __libc_start_main
> >                0xad5e258d4c544155
> > 
> >     67.56%     0.00%  qemu-system-x86  perf                        [.] run_builtin                               
> >             |
> >             ---run_builtin
> >                main
> >                __libc_start_main
> >                0xad5e258d4c544155
> > 
> > Can you take a look? What I have, i.e. the patches from Jiri and Kan
> > moving stuff out of the way of the python binding and this patch is at
> > the tmp.perf/core branch in my git tree:
> > 
> >   git://git.kernel.org/pub/scm/linux/kernel/git/acme/linux.git
> 
> hum, tried this branch and got proper behaviour for both stdio and TUI,

Well, from what he explained to me, the proper behaviour was the one I
got, i.e. one event has call-graph=fp, and thus the perf_event_attr
fields for callchains will be set up as always, and the other, with
call-graph=no, will not ask the kernel for callchains, thus saving
space, but report _will_ show callchains for it, using the callchains
obtained for the first event.

So, "call-graph=no" doesn't mean you don't want callchains for a
particular events _if_ there is another event in the group for which
callchains is available.

But if "call-graph=no" for all events, then, yes, "no" means really
"no". :-)

I think we should use "call-graph=ref" to mean that no callchains should
be requested to the kernel infrastructure but that when doing the
report, use callchains available in some other event (perhaps would be
good to specify which one), while "call-graph=no" really means "no",
i.e. no callchains asked from the kernel for this event, and _no_
callchains to appear on report.

- Arnaldo

> attaching stdio output
> 
> the record command was:
>   $ ./perf record -e 'cpu/cpu-cycles,call-graph=fp,time,period=10000/pp,cpu/instructions,call-graph=no,time=0,period=20000/p' ls
> 
> got the events strings from your report output.. what was different in your record command?
> 
> jirka

> # To display the perf.data header info, please use --header/--header-only options.
> #
> #
> # Total Lost Samples: 4
> #
> # Samples: 95  of event 'cpu/cpu-cycles,call-graph=fp,time,period=10000/pp'
> # Event count (approx.): 7487606
> #
> # Children      Self  Command  Shared Object       Symbol                               
> # ........  ........  .......  ..................  .....................................
> #
>     61.66%     0.00%  ls       [kernel.vmlinux]    [k] entry_SYSCALL_64_fastpath        
>                  |
>                  ---entry_SYSCALL_64_fastpath
>                     |          
>                     |--55.19%-- open64
>                     |          0x686361632e6f732e
>                     |          
>                     |--21.81%-- __GI___libc_write
>                     |          |          
>                     |          |--24.74%-- 0x6e69746c69756220
>                     |          |          
>                     |          |--19.20%-- 0x909632e66666964
>                     |          |          
>                     |          |--18.35%-- 0x657461746f6e6e61
>                     |          |          
>                     |          |--16.05%-- 0x1dbdc70
>                     |          |          
>                     |          |--11.32%-- 0
>                     |          |          
>                     |           --10.34%-- 0x9096f2e61746164
>                     |          
>                     |--6.18%-- __getdents64
>                     |          
>                     |--5.00%-- mmap64
>                     |          |          
>                     |          |--50.39%-- _dl_check_all_versions
>                     |          |          
>                     |           --49.61%-- _dl_map_object
>                     |                     0x213988
>                     |                     0
>                     |          
>                     |--2.72%-- mprotect
>                     |          dl_main
>                     |          _dl_sysdep_start
>                     |          
>                     |--1.26%-- _exit
>                     |          __run_exit_handlers
>                     |          0
>                     |          
>                     |--0.92%-- __GI___munmap
>                      --6.92%-- [...]
> 
>     34.03%     0.00%  ls       [unknown]           [.] 0x686361632e6f732e               
>                  |
>                  ---0x686361632e6f732e
> 
>     34.03%     0.00%  ls       ld-2.18.so          [.] open64                           
>                  |
>                  ---open64
>                     0x686361632e6f732e
> 
>     34.03%     0.00%  ls       [kernel.vmlinux]    [k] sys_open                         
>                  |
>                  ---sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     34.03%     0.00%  ls       [kernel.vmlinux]    [k] do_sys_open                      
>                  |
>                  ---do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     34.03%     0.00%  ls       [kernel.vmlinux]    [k] do_filp_open                     
>                  |
>                  ---do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     34.03%     0.00%  ls       [kernel.vmlinux]    [k] path_openat                      
>                  |
>                  ---path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     34.03%     0.00%  ls       [kernel.vmlinux]    [k] link_path_walk                   
>                  |
>                  ---link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     34.03%     0.00%  ls       [kernel.vmlinux]    [k] inode_permission                 
>                  |
>                  ---inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     34.03%     0.00%  ls       [kernel.vmlinux]    [k] __inode_permission               
>                  |
>                  ---__inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     33.75%     0.00%  ls       [kernel.vmlinux]    [k] irq_work_interrupt               
>                  |
>                  ---irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     33.75%     0.00%  ls       [kernel.vmlinux]    [k] smp_irq_work_interrupt           
>                  |
>                  ---smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     33.75%     0.00%  ls       [kernel.vmlinux]    [k] irq_work_run                     
>                  |
>                  ---irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     33.75%     0.00%  ls       [kernel.vmlinux]    [k] irq_work_run_list                
>                  |
>                  ---irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     33.75%     0.00%  ls       [kernel.vmlinux]    [k] perf_duration_warn               
>                  |
>                  ---perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     33.75%     0.00%  ls       [kernel.vmlinux]    [k] printk                           
>                  |
>                  ---printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     33.75%     0.00%  ls       [kernel.vmlinux]    [k] vprintk_default                  
>                  |
>                  ---vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     33.75%     0.00%  ls       [kernel.vmlinux]    [k] vprintk_emit                     
>                  |
>                  ---vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     33.75%     0.00%  ls       [kernel.vmlinux]    [k] console_unlock                   
>                  |
>                  ---console_unlock
>                     vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     33.43%     0.00%  ls       [kernel.vmlinux]    [k] call_console_drivers.constprop.27
>                  |
>                  ---call_console_drivers.constprop.27
>                     console_unlock
>                     vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     33.43%     0.00%  ls       [kernel.vmlinux]    [k] univ8250_console_write           
>                  |
>                  ---univ8250_console_write
>                     call_console_drivers.constprop.27
>                     console_unlock
>                     vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     33.43%     0.00%  ls       [kernel.vmlinux]    [k] uart_console_write               
>                  |
>                  ---uart_console_write
>                     univ8250_console_write
>                     call_console_drivers.constprop.27
>                     console_unlock
>                     vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     33.43%     0.00%  ls       [kernel.vmlinux]    [k] serial8250_console_putchar       
>                  |
>                  ---serial8250_console_putchar
>                     uart_console_write
>                     univ8250_console_write
>                     call_console_drivers.constprop.27
>                     console_unlock
>                     vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     28.96%     0.00%  ls       [kernel.vmlinux]    [k] wait_for_xmitr                   
>                  |
>                  ---wait_for_xmitr
>                     serial8250_console_putchar
>                     uart_console_write
>                     univ8250_console_write
>                     call_console_drivers.constprop.27
>                     console_unlock
>                     vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     17.71%     0.00%  ls       libc-2.18.so        [.] __GI___libc_write                
>                  |
>                  ---__GI___libc_write
>                     |          
>                     |--16.81%-- 0x657461746f6e6e61
>                     |          
>                     |--10.79%-- 0x6e69746c69756220
>                     |          
>                     |--8.37%-- 0x909632e66666964
>                     |          
>                     |--7.00%-- 0x1dbdc70
>                     |          
>                     |--6.67%-- 0x96f2e68636e6562
>                     |          
>                     |--5.30%-- 0x2d6469646c697562
>                     |          
>                     |--4.93%-- 0
>                     |          
>                     |--4.51%-- 0x9096f2e61746164
>                     |          
>                     |--3.30%-- 0x6f2e7473696c7665
>                      --32.32%-- [...]
> 
>     14.14%     0.41%  ls       [kernel.vmlinux]    [k] __const_udelay                   
>                  |
>                  ---__const_udelay
>                     |          
>                     |--97.12%-- wait_for_xmitr
>                     |          serial8250_console_putchar
>                     |          uart_console_write
>                     |          univ8250_console_write
>                     |          call_console_drivers.constprop.27
>                     |          console_unlock
>                     |          vprintk_emit
>                     |          vprintk_default
>                     |          printk
>                     |          perf_duration_warn
>                     |          irq_work_run_list
>                     |          irq_work_run
>                     |          smp_irq_work_interrupt
>                     |          irq_work_interrupt
>                     |          __inode_permission
>                     |          inode_permission
>                     |          link_path_walk
>                     |          path_openat
>                     |          do_filp_open
>                     |          do_sys_open
>                     |          sys_open
>                     |          entry_SYSCALL_64_fastpath
>                     |          open64
>                     |          0x686361632e6f732e
>                     |          
>                      --2.88%-- serial8250_console_putchar
>                                uart_console_write
>                                univ8250_console_write
>                                call_console_drivers.constprop.27
>                                console_unlock
>                                vprintk_emit
>                                vprintk_default
>                                printk
>                                perf_duration_warn
>                                irq_work_run_list
>                                irq_work_run
>                                smp_irq_work_interrupt
>                                irq_work_interrupt
>                                __inode_permission
>                                inode_permission
>                                link_path_walk
>                                path_openat
>                                do_filp_open
>                                do_sys_open
>                                sys_open
>                                entry_SYSCALL_64_fastpath
>                                open64
>                                0x686361632e6f732e
> 
>     13.75%    13.75%  ls       [kernel.vmlinux]    [k] io_serial_in                     
>                  |
>                  ---io_serial_in
>                     wait_for_xmitr
>                     serial8250_console_putchar
>                     uart_console_write
>                     univ8250_console_write
>                     call_console_drivers.constprop.27
>                     console_unlock
>                     vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>     13.45%     0.00%  ls       [kernel.vmlinux]    [k] sys_write                        
>                  |
>                  ---sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     |          
>                     |--14.21%-- 0x6e69746c69756220
>                     |          
>                     |--11.03%-- 0x909632e66666964
>                     |          
>                     |--10.54%-- 0x657461746f6e6e61
>                     |          
>                     |--9.22%-- 0x1dbdc70
>                     |          
>                     |--6.50%-- 0
>                     |          
>                     |--5.94%-- 0x9096f2e61746164
>                      --42.57%-- [...]
> 
>     13.45%     0.00%  ls       [kernel.vmlinux]    [k] vfs_write                        
>                  |
>                  ---vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     |          
>                     |--14.21%-- 0x6e69746c69756220
>                     |          
>                     |--11.03%-- 0x909632e66666964
>                     |          
>                     |--10.54%-- 0x657461746f6e6e61
>                     |          
>                     |--9.22%-- 0x1dbdc70
>                     |          
>                     |--6.50%-- 0
>                     |          
>                     |--5.94%-- 0x9096f2e61746164
>                      --42.57%-- [...]
> 
>     13.45%     0.00%  ls       [kernel.vmlinux]    [k] __vfs_write                      
>                  |
>                  ---__vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     |          
>                     |--14.21%-- 0x6e69746c69756220
>                     |          
>                     |--11.03%-- 0x909632e66666964
>                     |          
>                     |--10.54%-- 0x657461746f6e6e61
>                     |          
>                     |--9.22%-- 0x1dbdc70
>                     |          
>                     |--6.50%-- 0
>                     |          
>                     |--5.94%-- 0x9096f2e61746164
>                      --42.57%-- [...]
> 
>     13.45%     0.00%  ls       [kernel.vmlinux]    [k] tty_write                        
>                  |
>                  ---tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     |          
>                     |--14.21%-- 0x6e69746c69756220
>                     |          
>                     |--11.03%-- 0x909632e66666964
>                     |          
>                     |--10.54%-- 0x657461746f6e6e61
>                     |          
>                     |--9.22%-- 0x1dbdc70
>                     |          
>                     |--6.50%-- 0
>                     |          
>                     |--5.94%-- 0x9096f2e61746164
>                      --42.57%-- [...]
> 
>     13.45%     0.00%  ls       [kernel.vmlinux]    [k] n_tty_write                      
>                  |
>                  ---n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     |          
>                     |--14.21%-- 0x6e69746c69756220
>                     |          
>                     |--11.03%-- 0x909632e66666964
>                     |          
>                     |--10.54%-- 0x657461746f6e6e61
>                     |          
>                     |--9.22%-- 0x1dbdc70
>                     |          
>                     |--6.50%-- 0
>                     |          
>                     |--5.94%-- 0x9096f2e61746164
>                      --42.57%-- [...]
> 
>     12.39%    12.39%  ls       [kernel.vmlinux]    [k] delay_tsc                        
>                  |
>                  ---delay_tsc
>                     |          
>                     |--88.16%-- __const_udelay
>                     |          wait_for_xmitr
>                     |          serial8250_console_putchar
>                     |          uart_console_write
>                     |          univ8250_console_write
>                     |          call_console_drivers.constprop.27
>                     |          console_unlock
>                     |          vprintk_emit
>                     |          vprintk_default
>                     |          printk
>                     |          perf_duration_warn
>                     |          irq_work_run_list
>                     |          irq_work_run
>                     |          smp_irq_work_interrupt
>                     |          irq_work_interrupt
>                     |          __inode_permission
>                     |          inode_permission
>                     |          link_path_walk
>                     |          path_openat
>                     |          do_filp_open
>                     |          do_sys_open
>                     |          sys_open
>                     |          entry_SYSCALL_64_fastpath
>                     |          open64
>                     |          0x686361632e6f732e
>                     |          
>                      --11.84%-- wait_for_xmitr
>                                serial8250_console_putchar
>                                uart_console_write
>                                univ8250_console_write
>                                call_console_drivers.constprop.27
>                                console_unlock
>                                vprintk_emit
>                                vprintk_default
>                                printk
>                                perf_duration_warn
>                                irq_work_run_list
>                                irq_work_run
>                                smp_irq_work_interrupt
>                                irq_work_interrupt
>                                __inode_permission
>                                inode_permission
>                                link_path_walk
>                                path_openat
>                                do_filp_open
>                                do_sys_open
>                                sys_open
>                                entry_SYSCALL_64_fastpath
>                                open64
>                                0x686361632e6f732e
> 
>      9.88%     0.00%  ls       [unknown]           [.] 0x0000000000841f0f               
>                  |
>                  ---0x841f0f
> 
>      9.16%     0.00%  ls       [kernel.vmlinux]    [k] pty_write                        
>                  |
>                  ---pty_write
>                     |          
>                     |--45.92%-- n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--72.50%-- 0x6e69746c69756220
>                     |          |          
>                     |           --27.50%-- 0x909632e66666964
>                     |          
>                     |--31.59%-- do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--69.57%-- 0x657461746f6e6e61
>                     |          |          
>                     |           --30.43%-- 0x1dbdc70
>                     |          
>                      --22.49%-- tty_put_char
>                                do_output_char
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
> 
>      9.16%     0.00%  ls       [kernel.vmlinux]    [k] tty_flip_buffer_push             
>                  |
>                  ---tty_flip_buffer_push
>                     pty_write
>                     |          
>                     |--45.92%-- n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--72.50%-- 0x6e69746c69756220
>                     |          |          
>                     |           --27.50%-- 0x909632e66666964
>                     |          
>                     |--31.59%-- do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--69.57%-- 0x657461746f6e6e61
>                     |          |          
>                     |           --30.43%-- 0x1dbdc70
>                     |          
>                      --22.49%-- tty_put_char
>                                do_output_char
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
> 
>      9.16%     0.00%  ls       [kernel.vmlinux]    [k] tty_schedule_flip                
>                  |
>                  ---tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     |          
>                     |--45.92%-- n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--72.50%-- 0x6e69746c69756220
>                     |          |          
>                     |           --27.50%-- 0x909632e66666964
>                     |          
>                     |--31.59%-- do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--69.57%-- 0x657461746f6e6e61
>                     |          |          
>                     |           --30.43%-- 0x1dbdc70
>                     |          
>                      --22.49%-- tty_put_char
>                                do_output_char
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
> 
>      9.16%     0.00%  ls       [kernel.vmlinux]    [k] queue_work_on                    
>                  |
>                  ---queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     |          
>                     |--45.92%-- n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--72.50%-- 0x6e69746c69756220
>                     |          |          
>                     |           --27.50%-- 0x909632e66666964
>                     |          
>                     |--31.59%-- do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--69.57%-- 0x657461746f6e6e61
>                     |          |          
>                     |           --30.43%-- 0x1dbdc70
>                     |          
>                      --22.49%-- tty_put_char
>                                do_output_char
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
> 
>      9.16%     0.00%  ls       [kernel.vmlinux]    [k] __queue_work                     
>                  |
>                  ---__queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     |          
>                     |--45.92%-- n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--72.50%-- 0x6e69746c69756220
>                     |          |          
>                     |           --27.50%-- 0x909632e66666964
>                     |          
>                     |--31.59%-- do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--69.57%-- 0x657461746f6e6e61
>                     |          |          
>                     |           --30.43%-- 0x1dbdc70
>                     |          
>                      --22.49%-- tty_put_char
>                                do_output_char
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
> 
>      9.16%     1.30%  ls       [kernel.vmlinux]    [k] insert_work                      
>                  |
>                  ---insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     |          
>                     |--45.92%-- n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--72.50%-- 0x6e69746c69756220
>                     |          |          
>                     |           --27.50%-- 0x909632e66666964
>                     |          
>                     |--31.59%-- do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--69.57%-- 0x657461746f6e6e61
>                     |          |          
>                     |           --30.43%-- 0x1dbdc70
>                     |          
>                      --22.49%-- tty_put_char
>                                do_output_char
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
> 
>      8.41%     0.00%  ls       [unknown]           [.] 0000000000000000                 
>                  |
>                  ---0
>                     |          
>                     |--15.30%-- 0x632e74617473
>                     |          
>                     |--12.88%-- 0
>                     |          
>                     |--12.01%-- 0x72616863656d6974
>                      --59.81%-- [...]
> 
>      7.92%     0.00%  ls       ls                  [.] 0xffffffffffc0e1cf               
>                  |
>                  ---0xe1cf
>                     0x841f0f
> 
>      6.97%     0.00%  ls       [kernel.vmlinux]    [k] wake_up_worker                   
>                  |
>                  ---wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     |          
>                     |--37.80%-- n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--72.50%-- 0x6e69746c69756220
>                     |          |          
>                     |           --27.50%-- 0x909632e66666964
>                     |          
>                     |--32.63%-- do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |           --100.00%-- 0x657461746f6e6e61
>                     |          
>                      --29.57%-- tty_put_char
>                                do_output_char
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
> 
>      6.97%     0.00%  ls       [kernel.vmlinux]    [k] wake_up_process                  
>                  |
>                  ---wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     |          
>                     |--37.80%-- n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--72.50%-- 0x6e69746c69756220
>                     |          |          
>                     |           --27.50%-- 0x909632e66666964
>                     |          
>                     |--32.63%-- do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |           --100.00%-- 0x657461746f6e6e61
>                     |          
>                      --29.57%-- tty_put_char
>                                do_output_char
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
> 
>      6.97%     0.00%  ls       [kernel.vmlinux]    [k] try_to_wake_up                   
>                  |
>                  ---try_to_wake_up
>                     wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     |          
>                     |--37.80%-- n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--72.50%-- 0x6e69746c69756220
>                     |          |          
>                     |           --27.50%-- 0x909632e66666964
>                     |          
>                     |--32.63%-- do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |           --100.00%-- 0x657461746f6e6e61
>                     |          
>                      --29.57%-- tty_put_char
>                                do_output_char
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
> 
>      6.97%     0.00%  ls       [kernel.vmlinux]    [k] ttwu_do_activate.constprop.92    
>                  |
>                  ---ttwu_do_activate.constprop.92
>                     try_to_wake_up
>                     wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     |          
>                     |--37.80%-- n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--72.50%-- 0x6e69746c69756220
>                     |          |          
>                     |           --27.50%-- 0x909632e66666964
>                     |          
>                     |--32.63%-- do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |           --100.00%-- 0x657461746f6e6e61
>                     |          
>                      --29.57%-- tty_put_char
>                                do_output_char
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
> 
>      5.94%     5.94%  ls       libc-2.18.so        [.] get_next_seq                     
>                  |
>                  ---get_next_seq
>                     0xe1cf
>                     0x841f0f
> 
>      5.80%     0.00%  ls       [kernel.vmlinux]    [k] __schedule                       
>                  |
>                  ---__schedule
>                     |          
>                     |--73.77%-- preempt_schedule_common
>                     |          _cond_resched
>                     |          mutex_lock
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--28.65%-- 0
>                     |          |          
>                     |          |--26.18%-- 0x9096f2e61746164
>                     |          |          
>                     |          |--24.86%-- 0x909632e66666964
>                     |          |          
>                     |           --20.31%-- 0x1dbdc70
>                     |          
>                      --26.23%-- schedule
>                                prepare_exit_to_usermode
>                                syscall_return_slowpath
>                                int_ret_from_sys_call
>                                __GI___libc_write
>                                |          
>                                |--61.62%-- 0x2d6469646c697562
>                                |          
>                                 --38.38%-- 0x6f2e7473696c7665
> 
>      5.75%     1.94%  ls       libc-2.18.so        [.] __getdents64                     
>                  |
>                  ---__getdents64
>                     |          
>                     |--33.68%-- 0x616863656d69742d
>                      --66.32%-- [...]
> 
>      5.04%     0.00%  ls       [kernel.vmlinux]    [k] 0x000000007f47cb44               
>                  |
>                  ---0x27cb44
>                     entry_SYSCALL_64_fastpath
>                     |          
>                     |--15.40%-- _exit
>                     |          __run_exit_handlers
>                     |          0
>                      --84.60%-- [...]
> 
>      5.04%     0.00%  ls       [kernel.vmlinux]    [k] do_group_exit                    
>                  |
>                  ---do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
>                     |          
>                     |--15.40%-- _exit
>                     |          __run_exit_handlers
>                     |          0
>                      --84.60%-- [...]
> 
>      5.04%     0.78%  ls       [kernel.vmlinux]    [k] do_exit                          
>                  |
>                  ---do_exit
>                     do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
>                     |          
>                     |--15.40%-- _exit
>                     |          __run_exit_handlers
>                     |          0
>                      --84.60%-- [...]
> 
>      4.96%     0.00%  ls       [kernel.vmlinux]    [k] do_output_char                   
>                  |
>                  ---do_output_char
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     |          
>                     |--28.59%-- 0x657461746f6e6e61
>                     |          
>                     |--12.51%-- 0x1dbdc70
>                      --58.90%-- [...]
> 
>      4.79%     4.79%  ls       [kernel.vmlinux]    [k] native_write_msr_safe            
>                  |
>                  ---native_write_msr_safe
>                     |          
>                     |--37.79%-- __intel_pmu_disable_all
>                     |          intel_pmu_disable_all
>                     |          x86_pmu_disable
>                     |          perf_pmu_disable
>                     |          ctx_sched_out
>                     |          __perf_event_task_sched_out
>                     |          __schedule
>                     |          |          
>                     |          |--51.77%-- schedule
>                     |          |          prepare_exit_to_usermode
>                     |          |          syscall_return_slowpath
>                     |          |          int_ret_from_sys_call
>                     |          |          __GI___libc_write
>                     |          |          0x2d6469646c697562
>                     |          |          
>                     |           --48.23%-- preempt_schedule_common
>                     |                     _cond_resched
>                     |                     mutex_lock
>                     |                     n_tty_write
>                     |                     tty_write
>                     |                     __vfs_write
>                     |                     vfs_write
>                     |                     sys_write
>                     |                     entry_SYSCALL_64_fastpath
>                     |                     __GI___libc_write
>                     |                     0
>                     |          
>                     |--32.55%-- __GI___libc_write
>                     |          0x657461746f6e6e61
>                     |          
>                     |--29.58%-- __intel_pmu_enable_all.isra.9
>                     |          intel_pmu_enable_all
>                     |          x86_pmu_enable
>                     |          perf_pmu_enable
>                     |          |          
>                     |          |--56.32%-- perf_event_context_sched_in
>                     |          |          |          
>                     |          |          |--100.00%-- __perf_event_task_sched_in
>                     |          |          |          finish_task_switch
>                     |          |          |          __schedule
>                     |          |          |          preempt_schedule_common
>                     |          |          |          _cond_resched
>                     |          |          |          mutex_lock
>                     |          |          |          n_tty_write
>                     |          |          |          tty_write
>                     |          |          |          __vfs_write
>                     |          |          |          vfs_write
>                     |          |          |          sys_write
>                     |          |          |          entry_SYSCALL_64_fastpath
>                     |          |          |          __GI___libc_write
>                     |          |          |          0x9096f2e61746164
>                     |          |           --0.00%-- [...]
>                     |          |          
>                     |           --43.68%-- perf_pmu_sched_task
>                     |                     __perf_event_task_sched_out
>                     |                     __schedule
>                     |                     preempt_schedule_common
>                     |                     _cond_resched
>                     |                     mutex_lock
>                     |                     n_tty_write
>                     |                     tty_write
>                     |                     __vfs_write
>                     |                     vfs_write
>                     |                     sys_write
>                     |                     entry_SYSCALL_64_fastpath
>                     |                     __GI___libc_write
>                     |                     0x1dbdc70
>                      --0.08%-- [...]
> 
>      4.28%     0.00%  ls       [kernel.vmlinux]    [k] mutex_lock                       
>                  |
>                  ---mutex_lock
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     |          
>                     |--20.42%-- 0
>                     |          
>                     |--18.66%-- 0x9096f2e61746164
>                     |          
>                     |--17.71%-- 0x909632e66666964
>                     |          
>                     |--14.47%-- 0x1dbdc70
>                      --28.74%-- [...]
> 
>      4.28%     0.00%  ls       [kernel.vmlinux]    [k] _cond_resched                    
>                  |
>                  ---_cond_resched
>                     mutex_lock
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     |          
>                     |--20.42%-- 0
>                     |          
>                     |--18.66%-- 0x9096f2e61746164
>                     |          
>                     |--17.71%-- 0x909632e66666964
>                     |          
>                     |--14.47%-- 0x1dbdc70
>                      --28.74%-- [...]
> 
>      4.28%     0.00%  ls       [kernel.vmlinux]    [k] preempt_schedule_common          
>                  |
>                  ---preempt_schedule_common
>                     _cond_resched
>                     mutex_lock
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     |          
>                     |--20.42%-- 0
>                     |          
>                     |--18.66%-- 0x9096f2e61746164
>                     |          
>                     |--17.71%-- 0x909632e66666964
>                     |          
>                     |--14.47%-- 0x1dbdc70
>                      --28.74%-- [...]
> 
>      4.06%     4.06%  ls       [kernel.vmlinux]    [k] io_serial_out                    
>                  |
>                  ---io_serial_out
>                     serial8250_console_putchar
>                     uart_console_write
>                     univ8250_console_write
>                     call_console_drivers.constprop.27
>                     console_unlock
>                     vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>      4.04%     4.04%  ls       ls                  [.] 0x0000000000006686               
>                  |
>                  ---0x6686
> 
>      4.04%     0.00%  ls       ls                  [.] 0xffffffffffc06686               
>                  |
>                  ---0x6686
> 
>      3.81%     0.00%  ls       [kernel.vmlinux]    [k] sys_getdents                     
>                  |
>                  ---sys_getdents
>                     entry_SYSCALL_64_fastpath
>                     __getdents64
> 
>      3.81%     0.00%  ls       [kernel.vmlinux]    [k] iterate_dir                      
>                  |
>                  ---iterate_dir
>                     sys_getdents
>                     entry_SYSCALL_64_fastpath
>                     __getdents64
> 
>      3.81%     0.00%  ls       [ext4]              [k] ext4_readdir                     
>                  |
>                  ---ext4_readdir
>                     iterate_dir
>                     sys_getdents
>                     entry_SYSCALL_64_fastpath
>                     __getdents64
> 
>      3.81%     0.00%  ls       [ext4]              [k] ext4_htree_fill_tree             
>                  |
>                  ---ext4_htree_fill_tree
>                     ext4_readdir
>                     iterate_dir
>                     sys_getdents
>                     entry_SYSCALL_64_fastpath
>                     __getdents64
> 
>      3.81%     0.00%  ls       [ext4]              [k] htree_dirblock_to_tree           
>                  |
>                  ---htree_dirblock_to_tree
>                     ext4_htree_fill_tree
>                     ext4_readdir
>                     iterate_dir
>                     sys_getdents
>                     entry_SYSCALL_64_fastpath
>                     __getdents64
> 
>      3.64%     0.00%  ls       [kernel.vmlinux]    [k] activate_task                    
>                  |
>                  ---activate_task
>                     ttwu_do_activate.constprop.92
>                     try_to_wake_up
>                     wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     |          
>                     |--56.57%-- tty_put_char
>                     |          do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          
>                     |--23.55%-- do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          
>                      --19.88%-- n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
>                                0x909632e66666964
> 
>      3.64%     0.00%  ls       [kernel.vmlinux]    [k] enqueue_task                     
>                  |
>                  ---enqueue_task
>                     activate_task
>                     ttwu_do_activate.constprop.92
>                     try_to_wake_up
>                     wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     |          
>                     |--56.57%-- tty_put_char
>                     |          do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          
>                     |--23.55%-- do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          
>                      --19.88%-- n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
>                                0x909632e66666964
> 
>      3.64%     0.00%  ls       [kernel.vmlinux]    [k] enqueue_task_fair                
>                  |
>                  ---enqueue_task_fair
>                     enqueue_task
>                     activate_task
>                     ttwu_do_activate.constprop.92
>                     try_to_wake_up
>                     wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     |          
>                     |--56.57%-- tty_put_char
>                     |          do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          
>                     |--23.55%-- do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          
>                      --19.88%-- n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
>                                0x909632e66666964
> 
>      3.38%     0.00%  ls       ld-2.18.so          [.] _dl_map_object                   
>                  |
>                  ---_dl_map_object
>                     |          
>                     |--45.27%-- 0x21eb10
>                     |          0
>                     |          
>                     |--45.27%-- 0x213988
>                     |          0
>                     |          
>                      --9.45%-- 0x21e640
>                                0
> 
>      3.27%     0.00%  ls       ld-2.18.so          [.] _dl_sysdep_start                 
>                  |
>                  ---_dl_sysdep_start
> 
>      3.27%     0.00%  ls       ld-2.18.so          [.] dl_main                          
>                  |
>                  ---dl_main
>                     _dl_sysdep_start
> 
>      3.08%     0.00%  ls       ld-2.18.so          [.] mmap64                           
>                  |
>                  ---mmap64
>                     |          
>                     |--50.39%-- _dl_check_all_versions
>                     |          
>                      --49.61%-- _dl_map_object
>                                0x213988
>                                0
> 
>      3.08%     0.00%  ls       [kernel.vmlinux]    [k] sys_mmap                         
>                  |
>                  ---sys_mmap
>                     entry_SYSCALL_64_fastpath
>                     mmap64
>                     |          
>                     |--50.39%-- _dl_check_all_versions
>                     |          
>                      --49.61%-- _dl_map_object
>                                0x213988
>                                0
> 
>      3.08%     0.00%  ls       [kernel.vmlinux]    [k] sys_mmap_pgoff                   
>                  |
>                  ---sys_mmap_pgoff
>                     sys_mmap
>                     entry_SYSCALL_64_fastpath
>                     mmap64
>                     |          
>                     |--50.39%-- _dl_check_all_versions
>                     |          
>                      --49.61%-- _dl_map_object
>                                0x213988
>                                0
> 
>      3.08%     0.00%  ls       [kernel.vmlinux]    [k] vm_mmap_pgoff                    
>                  |
>                  ---vm_mmap_pgoff
>                     sys_mmap_pgoff
>                     sys_mmap
>                     entry_SYSCALL_64_fastpath
>                     mmap64
>                     |          
>                     |--50.39%-- _dl_check_all_versions
>                     |          
>                      --49.61%-- _dl_map_object
>                                0x213988
>                                0
> 
>      3.08%     0.00%  ls       [kernel.vmlinux]    [k] do_mmap_pgoff                    
>                  |
>                  ---do_mmap_pgoff
>                     vm_mmap_pgoff
>                     sys_mmap_pgoff
>                     sys_mmap
>                     entry_SYSCALL_64_fastpath
>                     mmap64
>                     |          
>                     |--50.39%-- _dl_check_all_versions
>                     |          
>                      --49.61%-- _dl_map_object
>                                0x213988
>                                0
> 
>      3.08%     0.00%  ls       [kernel.vmlinux]    [k] mmap_region                      
>                  |
>                  ---mmap_region
>                     do_mmap_pgoff
>                     vm_mmap_pgoff
>                     sys_mmap_pgoff
>                     sys_mmap
>                     entry_SYSCALL_64_fastpath
>                     mmap64
>                     |          
>                     |--50.39%-- _dl_check_all_versions
>                     |          
>                      --49.61%-- _dl_map_object
>                                0x213988
>                                0
> 
>      3.03%     0.00%  ls       [kernel.vmlinux]    [k] mmput                            
>                  |
>                  ---mmput
>                     do_exit
>                     do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
> 
>      3.03%     0.00%  ls       [kernel.vmlinux]    [k] exit_mmap                        
>                  |
>                  ---exit_mmap
>                     mmput
>                     do_exit
>                     do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
> 
>      2.98%     0.00%  ls       [unknown]           [.] 0x657461746f6e6e61               
>                  |
>                  ---0x657461746f6e6e61
> 
>      2.79%     0.64%  ls       [kernel.vmlinux]    [k] finish_task_switch               
>                  |
>                  ---finish_task_switch
>                     __schedule
>                     preempt_schedule_common
>                     _cond_resched
>                     mutex_lock
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     |          
>                     |--28.65%-- 0x9096f2e61746164
>                     |          
>                     |--27.20%-- 0x909632e66666964
>                      --44.14%-- [...]
> 
>      2.70%     0.00%  ls       [kernel.vmlinux]    [k] int_ret_from_sys_call            
>                  |
>                  ---int_ret_from_sys_call
>                     __GI___libc_write
>                     |          
>                     |--43.71%-- 0x96f2e68636e6562
>                     |          
>                     |--34.69%-- 0x2d6469646c697562
>                     |          
>                      --21.61%-- 0x6f2e7473696c7665
> 
>      2.70%     0.00%  ls       [kernel.vmlinux]    [k] syscall_return_slowpath          
>                  |
>                  ---syscall_return_slowpath
>                     int_ret_from_sys_call
>                     __GI___libc_write
>                     |          
>                     |--43.71%-- 0x96f2e68636e6562
>                     |          
>                     |--34.69%-- 0x2d6469646c697562
>                     |          
>                      --21.61%-- 0x6f2e7473696c7665
> 
>      2.70%     1.18%  ls       [kernel.vmlinux]    [k] prepare_exit_to_usermode         
>                  |
>                  ---prepare_exit_to_usermode
>                     syscall_return_slowpath
>                     int_ret_from_sys_call
>                     __GI___libc_write
>                     |          
>                     |--43.71%-- 0x96f2e68636e6562
>                     |          
>                     |--34.69%-- 0x2d6469646c697562
>                     |          
>                      --21.61%-- 0x6f2e7473696c7665
> 
>      2.43%     0.00%  ls       [kernel.vmlinux]    [k] __perf_event_task_sched_out      
>                  |
>                  ---__perf_event_task_sched_out
>                     __schedule
>                     |          
>                     |--61.42%-- preempt_schedule_common
>                     |          _cond_resched
>                     |          mutex_lock
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |          |--58.52%-- 0
>                     |          |          
>                     |           --41.48%-- 0x1dbdc70
>                     |          
>                      --38.58%-- schedule
>                                prepare_exit_to_usermode
>                                syscall_return_slowpath
>                                int_ret_from_sys_call
>                                __GI___libc_write
>                                0x2d6469646c697562
> 
>      2.30%     0.00%  ls       ls                  [.] 0xffffffffffc06dcb               
>                  |
>                  ---0x6dcb
>                     0
>                     |          
>                     |--56.03%-- 0x632e74617473
>                     |          
>                      --43.97%-- 0x72616863656d6974
> 
>      2.26%     1.63%  ls       ld-2.18.so          [.] do_lookup_x                      
>                  |
>                  ---do_lookup_x
> 
>      2.16%     1.54%  ls       [kernel.vmlinux]    [k] nmi                              
>                  |
>                  ---nmi
>                     |          
>                     |--63.15%-- __const_udelay
>                     |          wait_for_xmitr
>                     |          serial8250_console_putchar
>                     |          uart_console_write
>                     |          univ8250_console_write
>                     |          call_console_drivers.constprop.27
>                     |          console_unlock
>                     |          vprintk_emit
>                     |          vprintk_default
>                     |          printk
>                     |          perf_duration_warn
>                     |          irq_work_run_list
>                     |          irq_work_run
>                     |          smp_irq_work_interrupt
>                     |          irq_work_interrupt
>                     |          __inode_permission
>                     |          inode_permission
>                     |          link_path_walk
>                     |          path_openat
>                     |          do_filp_open
>                     |          do_sys_open
>                     |          sys_open
>                     |          entry_SYSCALL_64_fastpath
>                     |          open64
>                     |          0x686361632e6f732e
>                     |          
>                     |--29.07%-- do_lookup_x
>                     |          
>                     |--5.52%-- end_repeat_nmi
>                     |          scheduler_tick
>                     |          update_process_times
>                     |          tick_sched_handle.isra.17
>                     |          tick_sched_timer
>                     |          __hrtimer_run_queues
>                     |          hrtimer_interrupt
>                     |          local_apic_timer_interrupt
>                     |          smp_apic_timer_interrupt
>                     |          apic_timer_interrupt
>                     |          setup_new_exec
>                     |          load_elf_binary
>                     |          search_binary_handler
>                     |          do_execveat_common.isra.32
>                     |          sys_execve
>                     |          return_from_execve
>                     |          0x7f9abb498a77
>                     |          
>                     |--1.10%-- nmi_handle
>                     |          default_do_nmi
>                     |          do_nmi
>                     |          end_repeat_nmi
>                     |          scheduler_tick
>                     |          update_process_times
>                     |          tick_sched_handle.isra.17
>                     |          tick_sched_timer
>                     |          __hrtimer_run_queues
>                     |          hrtimer_interrupt
>                     |          local_apic_timer_interrupt
>                     |          smp_apic_timer_interrupt
>                     |          apic_timer_interrupt
>                     |          setup_new_exec
>                     |          load_elf_binary
>                     |          search_binary_handler
>                     |          do_execveat_common.isra.32
>                     |          sys_execve
>                     |          return_from_execve
>                     |          0x7f9abb498a77
>                     |          
>                     |--0.63%-- perf_event_nmi_handler
>                     |          nmi_handle
>                     |          default_do_nmi
>                     |          do_nmi
>                     |          end_repeat_nmi
>                     |          scheduler_tick
>                     |          update_process_times
>                     |          tick_sched_handle.isra.17
>                     |          tick_sched_timer
>                     |          __hrtimer_run_queues
>                     |          hrtimer_interrupt
>                     |          local_apic_timer_interrupt
>                     |          smp_apic_timer_interrupt
>                     |          apic_timer_interrupt
>                     |          setup_new_exec
>                     |          load_elf_binary
>                     |          search_binary_handler
>                     |          do_execveat_common.isra.32
>                     |          sys_execve
>                     |          return_from_execve
>                     |          0x7f9abb498a77
>                     |          
>                      --0.52%-- scheduler_tick
>                                update_process_times
>                                tick_sched_handle.isra.17
>                                tick_sched_timer
>                                __hrtimer_run_queues
>                                hrtimer_interrupt
>                                local_apic_timer_interrupt
>                                smp_apic_timer_interrupt
>                                apic_timer_interrupt
>                                setup_new_exec
>                                load_elf_binary
>                                search_binary_handler
>                                do_execveat_common.isra.32
>                                sys_execve
>                                return_from_execve
>                                0x7f9abb498a77
> 
>      2.15%     0.00%  ls       [kernel.vmlinux]    [k] __perf_event_task_sched_in       
>                  |
>                  ---__perf_event_task_sched_in
>                     finish_task_switch
>                     __schedule
>                     preempt_schedule_common
>                     _cond_resched
>                     mutex_lock
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     |          
>                     |--37.19%-- 0x9096f2e61746164
>                     |          
>                     |--35.31%-- 0x909632e66666964
>                      --27.50%-- [...]
> 
>      2.06%     0.00%  ls       [kernel.vmlinux]    [k] tty_put_char                     
>                  |
>                  ---tty_put_char
>                     do_output_char
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
> 
>      2.06%     2.06%  ls       [kernel.vmlinux]    [k] update_rq_runnable_avg           
>                  |
>                  ---update_rq_runnable_avg
>                     enqueue_task_fair
>                     enqueue_task
>                     activate_task
>                     ttwu_do_activate.constprop.92
>                     try_to_wake_up
>                     wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     tty_put_char
>                     do_output_char
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
> 
>      1.98%     1.98%  ls       libc-2.18.so        [.] __strcoll_l                      
>                  |
>                  ---__strcoll_l
>                     0xe1cf
>                     0x841f0f
> 
>      1.97%     1.35%  ls       [kernel.vmlinux]    [k] perf_pmu_sched_task              
>                  |
>                  ---perf_pmu_sched_task
>                     |          
>                     |--68.52%-- __perf_event_task_sched_in
>                     |          finish_task_switch
>                     |          __schedule
>                     |          preempt_schedule_common
>                     |          _cond_resched
>                     |          mutex_lock
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          |          
>                     |           --100.00%-- 0x909632e66666964
>                     |          
>                      --31.48%-- __perf_event_task_sched_out
>                                __schedule
>                                preempt_schedule_common
>                                _cond_resched
>                                mutex_lock
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
>                                0x1dbdc70
> 
>      1.95%     1.95%  ls       ls                  [.] 0x000000000000e270               
>                  |
>                  ---0xe270
>                     0x841f0f
> 
>      1.95%     0.00%  ls       ls                  [.] 0xffffffffffc0e270               
>                  |
>                  ---0xe270
>                     0x841f0f
> 
>      1.94%     0.00%  ls       [unknown]           [.] 0x616863656d69742d               
>                  |
>                  ---0x616863656d69742d
> 
>      1.93%     0.00%  ls       [kernel.vmlinux]    [k] unmap_vmas                       
>                  |
>                  ---unmap_vmas
>                     exit_mmap
>                     mmput
>                     do_exit
>                     do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
> 
>      1.93%     0.93%  ls       [kernel.vmlinux]    [k] unmap_single_vma                 
>                  |
>                  ---unmap_single_vma
>                     unmap_vmas
>                     exit_mmap
>                     mmput
>                     do_exit
>                     do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
> 
>      1.93%     0.00%  ls       [ext4]              [k] ext4fs_dirhash                   
>                  |
>                  ---ext4fs_dirhash
>                     htree_dirblock_to_tree
>                     ext4_htree_fill_tree
>                     ext4_readdir
>                     iterate_dir
>                     sys_getdents
>                     entry_SYSCALL_64_fastpath
>                     __getdents64
> 
>      1.93%     1.93%  ls       [ext4]              [k] str2hashbuf_signed               
>                  |
>                  ---str2hashbuf_signed
>                     ext4fs_dirhash
>                     htree_dirblock_to_tree
>                     ext4_htree_fill_tree
>                     ext4_readdir
>                     iterate_dir
>                     sys_getdents
>                     entry_SYSCALL_64_fastpath
>                     __getdents64
> 
>      1.91%     0.00%  ls       [unknown]           [.] 0x6e69746c69756220               
>                  |
>                  ---0x6e69746c69756220
> 
>      1.91%     0.00%  ls       [kernel.vmlinux]    [k] ttwu_do_wakeup                   
>                  |
>                  ---ttwu_do_wakeup
>                     ttwu_do_activate.constprop.92
>                     try_to_wake_up
>                     wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     0x6e69746c69756220
> 
>      1.91%     0.00%  ls       [kernel.vmlinux]    [k] check_preempt_curr               
>                  |
>                  ---check_preempt_curr
>                     ttwu_do_wakeup
>                     ttwu_do_activate.constprop.92
>                     try_to_wake_up
>                     wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     0x6e69746c69756220
> 
>      1.91%     0.00%  ls       [kernel.vmlinux]    [k] check_preempt_wakeup             
>                  |
>                  ---check_preempt_wakeup
>                     check_preempt_curr
>                     ttwu_do_wakeup
>                     ttwu_do_activate.constprop.92
>                     try_to_wake_up
>                     wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     0x6e69746c69756220
> 
>      1.91%     1.91%  ls       [kernel.vmlinux]    [k] set_next_buddy                   
>                  |
>                  ---set_next_buddy
>                     check_preempt_wakeup
>                     check_preempt_curr
>                     ttwu_do_wakeup
>                     ttwu_do_activate.constprop.92
>                     try_to_wake_up
>                     wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     0x6e69746c69756220
> 
>      1.89%     1.89%  ls       [ext4]              [k] ext4_htree_store_dirent          
>                  |
>                  ---ext4_htree_store_dirent
>                     htree_dirblock_to_tree
>                     ext4_htree_fill_tree
>                     ext4_readdir
>                     iterate_dir
>                     sys_getdents
>                     entry_SYSCALL_64_fastpath
>                     __getdents64
> 
>      1.85%     1.85%  ls       libc-2.18.so        [.] __strcmp_sse2                    
>                  |
>                  ---__strcmp_sse2
> 
>      1.82%     0.00%  ls       [unknown]           [.] 0x6f74207367756220               
>                  |
>                  ---0x6f74207367756220
> 
>      1.82%     1.82%  ls       libc-2.18.so        [.] strlen                           
>                  |
>                  ---strlen
>                     0x6f74207367756220
> 
>      1.81%     0.00%  ls       [kernel.vmlinux]    [k] ctx_sched_out                    
>                  |
>                  ---ctx_sched_out
>                     __perf_event_task_sched_out
>                     __schedule
>                     |          
>                     |--51.77%-- schedule
>                     |          prepare_exit_to_usermode
>                     |          syscall_return_slowpath
>                     |          int_ret_from_sys_call
>                     |          __GI___libc_write
>                     |          0x2d6469646c697562
>                     |          
>                      --48.23%-- preempt_schedule_common
>                                _cond_resched
>                                mutex_lock
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
>                                0
> 
>      1.81%     0.00%  ls       [kernel.vmlinux]    [k] perf_pmu_disable                 
>                  |
>                  ---perf_pmu_disable
>                     ctx_sched_out
>                     __perf_event_task_sched_out
>                     __schedule
>                     |          
>                     |--51.77%-- schedule
>                     |          prepare_exit_to_usermode
>                     |          syscall_return_slowpath
>                     |          int_ret_from_sys_call
>                     |          __GI___libc_write
>                     |          0x2d6469646c697562
>                     |          
>                      --48.23%-- preempt_schedule_common
>                                _cond_resched
>                                mutex_lock
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
>                                0
> 
>      1.81%     0.00%  ls       [kernel.vmlinux]    [k] x86_pmu_disable                  
>                  |
>                  ---x86_pmu_disable
>                     perf_pmu_disable
>                     ctx_sched_out
>                     __perf_event_task_sched_out
>                     __schedule
>                     |          
>                     |--51.77%-- schedule
>                     |          prepare_exit_to_usermode
>                     |          syscall_return_slowpath
>                     |          int_ret_from_sys_call
>                     |          __GI___libc_write
>                     |          0x2d6469646c697562
>                     |          
>                      --48.23%-- preempt_schedule_common
>                                _cond_resched
>                                mutex_lock
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
>                                0
> 
>      1.81%     0.00%  ls       [kernel.vmlinux]    [k] intel_pmu_disable_all            
>                  |
>                  ---intel_pmu_disable_all
>                     x86_pmu_disable
>                     perf_pmu_disable
>                     ctx_sched_out
>                     __perf_event_task_sched_out
>                     __schedule
>                     |          
>                     |--51.77%-- schedule
>                     |          prepare_exit_to_usermode
>                     |          syscall_return_slowpath
>                     |          int_ret_from_sys_call
>                     |          __GI___libc_write
>                     |          0x2d6469646c697562
>                     |          
>                      --48.23%-- preempt_schedule_common
>                                _cond_resched
>                                mutex_lock
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
>                                0
> 
>      1.81%     0.00%  ls       [kernel.vmlinux]    [k] __intel_pmu_disable_all          
>                  |
>                  ---__intel_pmu_disable_all
>                     intel_pmu_disable_all
>                     x86_pmu_disable
>                     perf_pmu_disable
>                     ctx_sched_out
>                     __perf_event_task_sched_out
>                     __schedule
>                     |          
>                     |--51.77%-- schedule
>                     |          prepare_exit_to_usermode
>                     |          syscall_return_slowpath
>                     |          int_ret_from_sys_call
>                     |          __GI___libc_write
>                     |          0x2d6469646c697562
>                     |          
>                      --48.23%-- preempt_schedule_common
>                                _cond_resched
>                                mutex_lock
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
>                                0
> 
>      1.76%     0.00%  ls       [unknown]           [.] 0x00401f0fc3f30678               
>                  |
>                  ---0x401f0fc3f30678
> 
>      1.76%     1.76%  ls       libc-2.18.so        [.] _dl_addr                         
>                  |
>                  ---_dl_addr
>                     0x401f0fc3f30678
> 
>      1.72%     0.00%  ls       ls                  [.] 0xffffffffffc06438               
>                  |
>                  ---0x6438
> 
>      1.72%     1.72%  ls       ls                  [.] 0x0000000000005b4d               
>                  |
>                  ---0x5b4d
>                     0x6438
> 
>      1.72%     0.00%  ls       ls                  [.] 0xffffffffffc05b4d               
>                  |
>                  ---0x5b4d
>                     0x6438
> 
>      1.70%     1.70%  ls       libpthread-2.18.so  [.] frame_dummy                      
>                  |
>                  ---frame_dummy
> 
>      1.68%     0.00%  ls       ld-2.18.so          [.] mprotect                         
>                  |
>                  ---mprotect
>                     dl_main
>                     _dl_sysdep_start
> 
>      1.68%     0.00%  ls       [kernel.vmlinux]    [k] sys_mprotect                     
>                  |
>                  ---sys_mprotect
>                     entry_SYSCALL_64_fastpath
>                     mprotect
>                     dl_main
>                     _dl_sysdep_start
> 
>      1.68%     0.00%  ls       [kernel.vmlinux]    [k] mprotect_fixup                   
>                  |
>                  ---mprotect_fixup
>                     sys_mprotect
>                     entry_SYSCALL_64_fastpath
>                     mprotect
>                     dl_main
>                     _dl_sysdep_start
> 
>      1.68%     0.00%  ls       [kernel.vmlinux]    [k] split_vma                        
>                  |
>                  ---split_vma
>                     mprotect_fixup
>                     sys_mprotect
>                     entry_SYSCALL_64_fastpath
>                     mprotect
>                     dl_main
>                     _dl_sysdep_start
> 
>      1.68%     0.00%  ls       [kernel.vmlinux]    [k] __split_vma.isra.34              
>                  |
>                  ---__split_vma.isra.34
>                     split_vma
>                     mprotect_fixup
>                     sys_mprotect
>                     entry_SYSCALL_64_fastpath
>                     mprotect
>                     dl_main
>                     _dl_sysdep_start
> 
>      1.68%     1.68%  ls       [kernel.vmlinux]    [k] vma_interval_tree_remove         
>                  |
>                  ---vma_interval_tree_remove
>                     __split_vma.isra.34
>                     split_vma
>                     mprotect_fixup
>                     sys_mprotect
>                     entry_SYSCALL_64_fastpath
>                     mprotect
>                     dl_main
>                     _dl_sysdep_start
> 
>      1.59%     1.59%  ls       ld-2.18.so          [.] _dl_relocate_object              
>                  |
>                  ---_dl_relocate_object
>                     dl_main
>                     _dl_sysdep_start
> 
>      1.58%     0.86%  ls       [kernel.vmlinux]    [k] enqueue_entity                   
>                  |
>                  ---enqueue_entity
>                     enqueue_task_fair
>                     enqueue_task
>                     activate_task
>                     ttwu_do_activate.constprop.92
>                     try_to_wake_up
>                     wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     |          
>                     |--54.22%-- do_output_char
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          
>                      --45.78%-- n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
>                                0x909632e66666964
> 
>      1.55%     0.00%  ls       ld-2.18.so          [.] _dl_check_all_versions           
>                  |
>                  ---_dl_check_all_versions
> 
>      1.55%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_mmap                  
>                  |
>                  ---perf_event_mmap
>                     mmap_region
>                     do_mmap_pgoff
>                     vm_mmap_pgoff
>                     sys_mmap_pgoff
>                     sys_mmap
>                     entry_SYSCALL_64_fastpath
>                     mmap64
>                     _dl_check_all_versions
> 
>      1.55%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_aux                   
>                  |
>                  ---perf_event_aux
>                     perf_event_mmap
>                     mmap_region
>                     do_mmap_pgoff
>                     vm_mmap_pgoff
>                     sys_mmap_pgoff
>                     sys_mmap
>                     entry_SYSCALL_64_fastpath
>                     mmap64
>                     _dl_check_all_versions
> 
>      1.55%     1.55%  ls       [kernel.vmlinux]    [k] perf_event_aux_ctx               
>                  |
>                  ---perf_event_aux_ctx
>                     perf_event_aux
>                     perf_event_mmap
>                     mmap_region
>                     do_mmap_pgoff
>                     vm_mmap_pgoff
>                     sys_mmap_pgoff
>                     sys_mmap
>                     entry_SYSCALL_64_fastpath
>                     mmap64
>                     _dl_check_all_versions
> 
>      1.53%     0.00%  ls       ld-2.18.so          [.] 0xffff804cabb57988               
>                  |
>                  ---0x213988
>                     0
> 
>      1.53%     0.00%  ls       ld-2.18.so          [.] 0xffff804cabb62b10               
>                  |
>                  ---0x21eb10
>                     0
> 
>      1.53%     0.00%  ls       ld-2.18.so          [.] memset                           
>                  |
>                  ---memset
>                     _dl_map_object
>                     0x21eb10
>                     0
> 
>      1.53%     0.00%  ls       [kernel.vmlinux]    [k] page_fault                       
>                  |
>                  ---page_fault
>                     memset
>                     _dl_map_object
>                     0x21eb10
>                     0
> 
>      1.53%     0.00%  ls       [kernel.vmlinux]    [k] do_page_fault                    
>                  |
>                  ---do_page_fault
>                     page_fault
>                     memset
>                     _dl_map_object
>                     0x21eb10
>                     0
> 
>      1.53%     0.00%  ls       [kernel.vmlinux]    [k] __do_page_fault                  
>                  |
>                  ---__do_page_fault
>                     do_page_fault
>                     page_fault
>                     memset
>                     _dl_map_object
>                     0x21eb10
>                     0
> 
>      1.53%     0.00%  ls       [kernel.vmlinux]    [k] handle_mm_fault                  
>                  |
>                  ---handle_mm_fault
>                     __do_page_fault
>                     do_page_fault
>                     page_fault
>                     memset
>                     _dl_map_object
>                     0x21eb10
>                     0
> 
>      1.53%     1.53%  ls       [kernel.vmlinux]    [k] put_page                         
>                  |
>                  ---put_page
>                     handle_mm_fault
>                     __do_page_fault
>                     do_page_fault
>                     page_fault
>                     memset
>                     _dl_map_object
>                     0x21eb10
>                     0
> 
>      1.53%     0.00%  ls       [kernel.vmlinux]    [k] vma_link                         
>                  |
>                  ---vma_link
>                     mmap_region
>                     do_mmap_pgoff
>                     vm_mmap_pgoff
>                     sys_mmap_pgoff
>                     sys_mmap
>                     entry_SYSCALL_64_fastpath
>                     mmap64
>                     _dl_map_object
>                     0x213988
>                     0
> 
>      1.53%     1.53%  ls       [kernel.vmlinux]    [k] vma_interval_tree_insert         
>                  |
>                  ---vma_interval_tree_insert
>                     vma_link
>                     mmap_region
>                     do_mmap_pgoff
>                     vm_mmap_pgoff
>                     sys_mmap_pgoff
>                     sys_mmap
>                     entry_SYSCALL_64_fastpath
>                     mmap64
>                     _dl_map_object
>                     0x213988
>                     0
> 
>      1.52%     0.00%  ls       [kernel.vmlinux]    [k] schedule                         
>                  |
>                  ---schedule
>                     prepare_exit_to_usermode
>                     syscall_return_slowpath
>                     int_ret_from_sys_call
>                     __GI___libc_write
>                     |          
>                     |--61.62%-- 0x2d6469646c697562
>                     |          
>                      --38.38%-- 0x6f2e7473696c7665
> 
>      1.48%     0.00%  ls       [unknown]           [.] 0x0909632e66666964               
>                  |
>                  ---0x909632e66666964
> 
>      1.44%     1.44%  ls       [kernel.vmlinux]    [k] paranoid_entry                   
>                  |
>                  ---paranoid_entry
>                     __const_udelay
>                     wait_for_xmitr
>                     serial8250_console_putchar
>                     uart_console_write
>                     univ8250_console_write
>                     call_console_drivers.constprop.27
>                     console_unlock
>                     vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>      1.42%     0.00%  ls       [kernel.vmlinux]    [k] perf_pmu_enable                  
>                  |
>                  ---perf_pmu_enable
>                     |          
>                     |--56.32%-- perf_event_context_sched_in
>                     |          |          
>                     |          |--100.00%-- __perf_event_task_sched_in
>                     |          |          finish_task_switch
>                     |          |          __schedule
>                     |          |          preempt_schedule_common
>                     |          |          _cond_resched
>                     |          |          mutex_lock
>                     |          |          n_tty_write
>                     |          |          tty_write
>                     |          |          __vfs_write
>                     |          |          vfs_write
>                     |          |          sys_write
>                     |          |          entry_SYSCALL_64_fastpath
>                     |          |          __GI___libc_write
>                     |          |          0x9096f2e61746164
>                     |           --0.00%-- [...]
>                     |          
>                      --43.68%-- perf_pmu_sched_task
>                                __perf_event_task_sched_out
>                                __schedule
>                                preempt_schedule_common
>                                _cond_resched
>                                mutex_lock
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
>                                0x1dbdc70
> 
>      1.42%     0.00%  ls       [kernel.vmlinux]    [k] x86_pmu_enable                   
>                  |
>                  ---x86_pmu_enable
>                     perf_pmu_enable
>                     |          
>                     |--56.32%-- perf_event_context_sched_in
>                     |          |          
>                     |          |--100.00%-- __perf_event_task_sched_in
>                     |          |          finish_task_switch
>                     |          |          __schedule
>                     |          |          preempt_schedule_common
>                     |          |          _cond_resched
>                     |          |          mutex_lock
>                     |          |          n_tty_write
>                     |          |          tty_write
>                     |          |          __vfs_write
>                     |          |          vfs_write
>                     |          |          sys_write
>                     |          |          entry_SYSCALL_64_fastpath
>                     |          |          __GI___libc_write
>                     |          |          0x9096f2e61746164
>                     |           --0.00%-- [...]
>                     |          
>                      --43.68%-- perf_pmu_sched_task
>                                __perf_event_task_sched_out
>                                __schedule
>                                preempt_schedule_common
>                                _cond_resched
>                                mutex_lock
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
>                                0x1dbdc70
> 
>      1.42%     0.00%  ls       [kernel.vmlinux]    [k] intel_pmu_enable_all             
>                  |
>                  ---intel_pmu_enable_all
>                     x86_pmu_enable
>                     perf_pmu_enable
>                     |          
>                     |--56.32%-- perf_event_context_sched_in
>                     |          |          
>                     |          |--100.00%-- __perf_event_task_sched_in
>                     |          |          finish_task_switch
>                     |          |          __schedule
>                     |          |          preempt_schedule_common
>                     |          |          _cond_resched
>                     |          |          mutex_lock
>                     |          |          n_tty_write
>                     |          |          tty_write
>                     |          |          __vfs_write
>                     |          |          vfs_write
>                     |          |          sys_write
>                     |          |          entry_SYSCALL_64_fastpath
>                     |          |          __GI___libc_write
>                     |          |          0x9096f2e61746164
>                     |           --0.00%-- [...]
>                     |          
>                      --43.68%-- perf_pmu_sched_task
>                                __perf_event_task_sched_out
>                                __schedule
>                                preempt_schedule_common
>                                _cond_resched
>                                mutex_lock
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
>                                0x1dbdc70
> 
>      1.42%     0.00%  ls       [kernel.vmlinux]    [k] __intel_pmu_enable_all.isra.9    
>                  |
>                  ---__intel_pmu_enable_all.isra.9
>                     intel_pmu_enable_all
>                     x86_pmu_enable
>                     perf_pmu_enable
>                     |          
>                     |--56.32%-- perf_event_context_sched_in
>                     |          |          
>                     |          |--100.00%-- __perf_event_task_sched_in
>                     |          |          finish_task_switch
>                     |          |          __schedule
>                     |          |          preempt_schedule_common
>                     |          |          _cond_resched
>                     |          |          mutex_lock
>                     |          |          n_tty_write
>                     |          |          tty_write
>                     |          |          __vfs_write
>                     |          |          vfs_write
>                     |          |          sys_write
>                     |          |          entry_SYSCALL_64_fastpath
>                     |          |          __GI___libc_write
>                     |          |          0x9096f2e61746164
>                     |           --0.00%-- [...]
>                     |          
>                      --43.68%-- perf_pmu_sched_task
>                                __perf_event_task_sched_out
>                                __schedule
>                                preempt_schedule_common
>                                _cond_resched
>                                mutex_lock
>                                n_tty_write
>                                tty_write
>                                __vfs_write
>                                vfs_write
>                                sys_write
>                                entry_SYSCALL_64_fastpath
>                                __GI___libc_write
>                                0x1dbdc70
> 
>      1.42%     0.00%  ls       [kernel.vmlinux]    [k] wq_worker_waking_up              
>                  |
>                  ---wq_worker_waking_up
>                     ttwu_do_activate.constprop.92
>                     try_to_wake_up
>                     wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     do_output_char
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     0x657461746f6e6e61
> 
>      1.42%     1.42%  ls       [kernel.vmlinux]    [k] kthread_data                     
>                  |
>                  ---kthread_data
>                     wq_worker_waking_up
>                     ttwu_do_activate.constprop.92
>                     try_to_wake_up
>                     wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     do_output_char
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     0x657461746f6e6e61
> 
>      1.31%     0.04%  ls       [kernel.vmlinux]    [k] do_nmi                           
>                  |
>                  ---do_nmi
>                     |          
>                     |--52.08%-- end_repeat_nmi
>                     |          |          
>                     |          |--46.71%-- vsnprintf
>                     |          |          sprintf
>                     |          |          print_time.part.12
>                     |          |          print_prefix
>                     |          |          msg_print_text
>                     |          |          console_unlock
>                     |          |          vprintk_emit
>                     |          |          vprintk_default
>                     |          |          printk
>                     |          |          perf_duration_warn
>                     |          |          irq_work_run_list
>                     |          |          irq_work_run
>                     |          |          smp_irq_work_interrupt
>                     |          |          irq_work_interrupt
>                     |          |          __inode_permission
>                     |          |          inode_permission
>                     |          |          link_path_walk
>                     |          |          path_openat
>                     |          |          do_filp_open
>                     |          |          do_sys_open
>                     |          |          sys_open
>                     |          |          entry_SYSCALL_64_fastpath
>                     |          |          open64
>                     |          |          0x686361632e6f732e
>                     |          |          
>                     |          |--41.63%-- __inode_permission
>                     |          |          inode_permission
>                     |          |          link_path_walk
>                     |          |          path_openat
>                     |          |          do_filp_open
>                     |          |          do_sys_open
>                     |          |          sys_open
>                     |          |          entry_SYSCALL_64_fastpath
>                     |          |          open64
>                     |          |          0x686361632e6f732e
>                     |          |          
>                     |           --11.66%-- scheduler_tick
>                     |                     update_process_times
>                     |                     tick_sched_handle.isra.17
>                     |                     tick_sched_timer
>                     |                     __hrtimer_run_queues
>                     |                     hrtimer_interrupt
>                     |                     local_apic_timer_interrupt
>                     |                     smp_apic_timer_interrupt
>                     |                     apic_timer_interrupt
>                     |                     setup_new_exec
>                     |                     load_elf_binary
>                     |                     search_binary_handler
>                     |                     do_execveat_common.isra.32
>                     |                     sys_execve
>                     |                     return_from_execve
>                     |                     0x7f9abb498a77
>                     |          
>                      --47.92%-- nmi
>                                do_lookup_x
> 
>      1.29%     0.00%  ls       [unknown]           [.] 0x0000632e74617473               
>                  |
>                  ---0x632e74617473
> 
>      1.29%     1.29%  ls       ls                  [.] 0x0000000000005a40               
>                  |
>                  ---0x5a40
>                     0x6dcb
>                     0
>                     0x632e74617473
> 
>      1.29%     0.00%  ls       ls                  [.] 0xffffffffffc05a40               
>                  |
>                  ---0x5a40
>                     0x6dcb
>                     0
>                     0x632e74617473
> 
>      1.27%     0.00%  ls       [kernel.vmlinux]    [k] default_do_nmi                   
>                  |
>                  ---default_do_nmi
>                     do_nmi
>                     |          
>                     |--50.48%-- end_repeat_nmi
>                     |          |          
>                     |          |--49.78%-- vsnprintf
>                     |          |          sprintf
>                     |          |          print_time.part.12
>                     |          |          print_prefix
>                     |          |          msg_print_text
>                     |          |          console_unlock
>                     |          |          vprintk_emit
>                     |          |          vprintk_default
>                     |          |          printk
>                     |          |          perf_duration_warn
>                     |          |          irq_work_run_list
>                     |          |          irq_work_run
>                     |          |          smp_irq_work_interrupt
>                     |          |          irq_work_interrupt
>                     |          |          __inode_permission
>                     |          |          inode_permission
>                     |          |          link_path_walk
>                     |          |          path_openat
>                     |          |          do_filp_open
>                     |          |          do_sys_open
>                     |          |          sys_open
>                     |          |          entry_SYSCALL_64_fastpath
>                     |          |          open64
>                     |          |          0x686361632e6f732e
>                     |          |          
>                     |          |--44.37%-- __inode_permission
>                     |          |          inode_permission
>                     |          |          link_path_walk
>                     |          |          path_openat
>                     |          |          do_filp_open
>                     |          |          do_sys_open
>                     |          |          sys_open
>                     |          |          entry_SYSCALL_64_fastpath
>                     |          |          open64
>                     |          |          0x686361632e6f732e
>                     |          |          
>                     |           --5.85%-- scheduler_tick
>                     |                     update_process_times
>                     |                     tick_sched_handle.isra.17
>                     |                     tick_sched_timer
>                     |                     __hrtimer_run_queues
>                     |                     hrtimer_interrupt
>                     |                     local_apic_timer_interrupt
>                     |                     smp_apic_timer_interrupt
>                     |                     apic_timer_interrupt
>                     |                     setup_new_exec
>                     |                     load_elf_binary
>                     |                     search_binary_handler
>                     |                     do_execveat_common.isra.32
>                     |                     sys_execve
>                     |                     return_from_execve
>                     |                     0x7f9abb498a77
>                     |          
>                      --49.52%-- nmi
>                                do_lookup_x
> 
>      1.27%     0.63%  ls       [kernel.vmlinux]    [k] nmi_handle                       
>                  |
>                  ---nmi_handle
>                     default_do_nmi
>                     do_nmi
>                     |          
>                     |--50.48%-- end_repeat_nmi
>                     |          |          
>                     |          |--49.78%-- vsnprintf
>                     |          |          sprintf
>                     |          |          print_time.part.12
>                     |          |          print_prefix
>                     |          |          msg_print_text
>                     |          |          console_unlock
>                     |          |          vprintk_emit
>                     |          |          vprintk_default
>                     |          |          printk
>                     |          |          perf_duration_warn
>                     |          |          irq_work_run_list
>                     |          |          irq_work_run
>                     |          |          smp_irq_work_interrupt
>                     |          |          irq_work_interrupt
>                     |          |          __inode_permission
>                     |          |          inode_permission
>                     |          |          link_path_walk
>                     |          |          path_openat
>                     |          |          do_filp_open
>                     |          |          do_sys_open
>                     |          |          sys_open
>                     |          |          entry_SYSCALL_64_fastpath
>                     |          |          open64
>                     |          |          0x686361632e6f732e
>                     |          |          
>                     |          |--44.37%-- __inode_permission
>                     |          |          inode_permission
>                     |          |          link_path_walk
>                     |          |          path_openat
>                     |          |          do_filp_open
>                     |          |          do_sys_open
>                     |          |          sys_open
>                     |          |          entry_SYSCALL_64_fastpath
>                     |          |          open64
>                     |          |          0x686361632e6f732e
>                     |          |          
>                     |           --5.85%-- scheduler_tick
>                     |                     update_process_times
>                     |                     tick_sched_handle.isra.17
>                     |                     tick_sched_timer
>                     |                     __hrtimer_run_queues
>                     |                     hrtimer_interrupt
>                     |                     local_apic_timer_interrupt
>                     |                     smp_apic_timer_interrupt
>                     |                     apic_timer_interrupt
>                     |                     setup_new_exec
>                     |                     load_elf_binary
>                     |                     search_binary_handler
>                     |                     do_execveat_common.isra.32
>                     |                     sys_execve
>                     |                     return_from_execve
>                     |                     0x7f9abb498a77
>                     |          
>                      --49.52%-- nmi
>                                do_lookup_x
> 
>      1.24%     0.00%  ls       [unknown]           [.] 0x0000000001dbdc70               
>                  |
>                  ---0x1dbdc70
> 
>      1.23%     0.00%  ls       [kernel.vmlinux]    [k] task_work_run                    
>                  |
>                  ---task_work_run
>                     do_exit
>                     do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
> 
>      1.23%     0.00%  ls       [kernel.vmlinux]    [k] ____fput                         
>                  |
>                  ---____fput
>                     task_work_run
>                     do_exit
>                     do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
> 
>      1.23%     0.00%  ls       [kernel.vmlinux]    [k] __fput                           
>                  |
>                  ---__fput
>                     ____fput
>                     task_work_run
>                     do_exit
>                     do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
> 
>      1.23%     1.23%  ls       [kernel.vmlinux]    [k] locks_remove_file                
>                  |
>                  ---locks_remove_file
>                     __fput
>                     ____fput
>                     task_work_run
>                     do_exit
>                     do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
> 
>      1.18%     0.00%  ls       [unknown]           [.] 0x096f2e68636e6562               
>                  |
>                  ---0x96f2e68636e6562
> 
>      1.10%     0.00%  ls       [kernel.vmlinux]    [k] tlb_finish_mmu                   
>                  |
>                  ---tlb_finish_mmu
>                     exit_mmap
>                     mmput
>                     do_exit
>                     do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
> 
>      1.10%     0.00%  ls       [kernel.vmlinux]    [k] tlb_flush_mmu_free               
>                  |
>                  ---tlb_flush_mmu_free
>                     tlb_finish_mmu
>                     exit_mmap
>                     mmput
>                     do_exit
>                     do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
> 
>      1.10%     0.00%  ls       [kernel.vmlinux]    [k] free_pages_and_swap_cache        
>                  |
>                  ---free_pages_and_swap_cache
>                     tlb_flush_mmu_free
>                     tlb_finish_mmu
>                     exit_mmap
>                     mmput
>                     do_exit
>                     do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
> 
>      1.10%     1.10%  ls       [kernel.vmlinux]    [k] release_pages                    
>                  |
>                  ---release_pages
>                     free_pages_and_swap_cache
>                     tlb_flush_mmu_free
>                     tlb_finish_mmu
>                     exit_mmap
>                     mmput
>                     do_exit
>                     do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
> 
>      1.08%     1.08%  ls       ls                  [.] 0x0000000000006b20               
>                  |
>                  ---0x6b20
>                     0
>                     0
> 
>      1.08%     0.00%  ls       ls                  [.] 0xffffffffffc06b20               
>                  |
>                  ---0x6b20
>                     0
>                     0
> 
>      1.01%     0.00%  ls       [unknown]           [.] 0x72616863656d6974               
>                  |
>                  ---0x72616863656d6974
> 
>      1.01%     1.01%  ls       ls                  [.] __ctype_get_mb_cur_max@plt       
>                  |
>                  ---__ctype_get_mb_cur_max@plt
>                     0x6dcb
>                     0
>                     0x72616863656d6974
> 
>      1.01%     1.01%  ls       [kernel.vmlinux]    [k] unmap_page_range                 
>                  |
>                  ---unmap_page_range
>                     unmap_single_vma
>                     unmap_vmas
>                     exit_mmap
>                     mmput
>                     do_exit
>                     do_group_exit
>                     0x27cb44
>                     entry_SYSCALL_64_fastpath
> 
>      0.94%     0.00%  ls       [unknown]           [.] 0x2d6469646c697562               
>                  |
>                  ---0x2d6469646c697562
> 
>      0.89%     0.89%  ls       [kernel.vmlinux]    [k] get_pwq.isra.18                  
>                  |
>                  ---get_pwq.isra.18
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
> 
>      0.80%     0.00%  ls       [kernel.vmlinux]    [k] end_repeat_nmi                   
>                  |
>                  ---end_repeat_nmi
>                     |          
>                     |--39.76%-- vsnprintf
>                     |          sprintf
>                     |          print_time.part.12
>                     |          print_prefix
>                     |          msg_print_text
>                     |          console_unlock
>                     |          vprintk_emit
>                     |          vprintk_default
>                     |          printk
>                     |          perf_duration_warn
>                     |          irq_work_run_list
>                     |          irq_work_run
>                     |          smp_irq_work_interrupt
>                     |          irq_work_interrupt
>                     |          __inode_permission
>                     |          inode_permission
>                     |          link_path_walk
>                     |          path_openat
>                     |          do_filp_open
>                     |          do_sys_open
>                     |          sys_open
>                     |          entry_SYSCALL_64_fastpath
>                     |          open64
>                     |          0x686361632e6f732e
>                     |          
>                     |--35.44%-- __inode_permission
>                     |          inode_permission
>                     |          link_path_walk
>                     |          path_openat
>                     |          do_filp_open
>                     |          do_sys_open
>                     |          sys_open
>                     |          entry_SYSCALL_64_fastpath
>                     |          open64
>                     |          0x686361632e6f732e
>                     |          
>                      --24.80%-- scheduler_tick
>                                update_process_times
>                                tick_sched_handle.isra.17
>                                tick_sched_timer
>                                __hrtimer_run_queues
>                                hrtimer_interrupt
>                                local_apic_timer_interrupt
>                                smp_apic_timer_interrupt
>                                apic_timer_interrupt
>                                setup_new_exec
>                                load_elf_binary
>                                search_binary_handler
>                                do_execveat_common.isra.32
>                                sys_execve
>                                return_from_execve
>                                0x7f9abb498a77
> 
>      0.80%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_context_sched_in      
>                  |
>                  ---perf_event_context_sched_in
>                     |          
>                     |--100.00%-- __perf_event_task_sched_in
>                     |          finish_task_switch
>                     |          __schedule
>                     |          preempt_schedule_common
>                     |          _cond_resched
>                     |          mutex_lock
>                     |          n_tty_write
>                     |          tty_write
>                     |          __vfs_write
>                     |          vfs_write
>                     |          sys_write
>                     |          entry_SYSCALL_64_fastpath
>                     |          __GI___libc_write
>                     |          0x9096f2e61746164
>                      --0.00%-- [...]
> 
>      0.80%     0.00%  ls       [unknown]           [.] 0x09096f2e61746164               
>                  |
>                  ---0x9096f2e61746164
> 
>      0.78%     0.00%  ls       libc-2.18.so        [.] __run_exit_handlers              
>                  |
>                  ---__run_exit_handlers
>                     0
> 
>      0.78%     0.00%  ls       libc-2.18.so        [.] _exit                            
>                  |
>                  ---_exit
>                     __run_exit_handlers
>                     0
> 
>      0.72%     0.72%  ls       [kernel.vmlinux]    [k] account_entity_enqueue           
>                  |
>                  ---account_entity_enqueue
>                     enqueue_entity
>                     enqueue_task_fair
>                     enqueue_task
>                     activate_task
>                     ttwu_do_activate.constprop.92
>                     try_to_wake_up
>                     wake_up_process
>                     wake_up_worker
>                     insert_work
>                     __queue_work
>                     queue_work_on
>                     tty_schedule_flip
>                     tty_flip_buffer_push
>                     pty_write
>                     n_tty_write
>                     tty_write
>                     __vfs_write
>                     vfs_write
>                     sys_write
>                     entry_SYSCALL_64_fastpath
>                     __GI___libc_write
>                     0x909632e66666964
> 
>      0.62%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_nmi_handler           
>                  |
>                  ---perf_event_nmi_handler
>                     nmi_handle
>                     default_do_nmi
>                     do_nmi
>                     end_repeat_nmi
>                     |          
>                     |--51.70%-- vsnprintf
>                     |          sprintf
>                     |          print_time.part.12
>                     |          print_prefix
>                     |          msg_print_text
>                     |          console_unlock
>                     |          vprintk_emit
>                     |          vprintk_default
>                     |          printk
>                     |          perf_duration_warn
>                     |          irq_work_run_list
>                     |          irq_work_run
>                     |          smp_irq_work_interrupt
>                     |          irq_work_interrupt
>                     |          __inode_permission
>                     |          inode_permission
>                     |          link_path_walk
>                     |          path_openat
>                     |          do_filp_open
>                     |          do_sys_open
>                     |          sys_open
>                     |          entry_SYSCALL_64_fastpath
>                     |          open64
>                     |          0x686361632e6f732e
>                     |          
>                     |--46.08%-- __inode_permission
>                     |          inode_permission
>                     |          link_path_walk
>                     |          path_openat
>                     |          do_filp_open
>                     |          do_sys_open
>                     |          sys_open
>                     |          entry_SYSCALL_64_fastpath
>                     |          open64
>                     |          0x686361632e6f732e
>                     |          
>                      --2.22%-- scheduler_tick
>                                update_process_times
>                                tick_sched_handle.isra.17
>                                tick_sched_timer
>                                __hrtimer_run_queues
>                                hrtimer_interrupt
>                                local_apic_timer_interrupt
>                                smp_apic_timer_interrupt
>                                apic_timer_interrupt
>                                setup_new_exec
>                                load_elf_binary
>                                search_binary_handler
>                                do_execveat_common.isra.32
>                                sys_execve
>                                return_from_execve
>                                0x7f9abb498a77
> 
>      0.60%     0.60%  ls       [kernel.vmlinux]    [k] perf_sample_event_took           
>                  |
>                  ---perf_sample_event_took
>                     perf_event_nmi_handler
>                     nmi_handle
>                     default_do_nmi
>                     do_nmi
>                     end_repeat_nmi
>                     |          
>                     |--52.87%-- vsnprintf
>                     |          sprintf
>                     |          print_time.part.12
>                     |          print_prefix
>                     |          msg_print_text
>                     |          console_unlock
>                     |          vprintk_emit
>                     |          vprintk_default
>                     |          printk
>                     |          perf_duration_warn
>                     |          irq_work_run_list
>                     |          irq_work_run
>                     |          smp_irq_work_interrupt
>                     |          irq_work_interrupt
>                     |          __inode_permission
>                     |          inode_permission
>                     |          link_path_walk
>                     |          path_openat
>                     |          do_filp_open
>                     |          do_sys_open
>                     |          sys_open
>                     |          entry_SYSCALL_64_fastpath
>                     |          open64
>                     |          0x686361632e6f732e
>                     |          
>                      --47.13%-- __inode_permission
>                                inode_permission
>                                link_path_walk
>                                path_openat
>                                do_filp_open
>                                do_sys_open
>                                sys_open
>                                entry_SYSCALL_64_fastpath
>                                open64
>                                0x686361632e6f732e
> 
>      0.58%     0.00%  ls       [unknown]           [.] 0x6f2e7473696c7665               
>                  |
>                  ---0x6f2e7473696c7665
> 
>      0.58%     0.00%  ls       [kernel.vmlinux]    [k] pick_next_task_fair              
>                  |
>                  ---pick_next_task_fair
>                     __schedule
>                     schedule
>                     prepare_exit_to_usermode
>                     syscall_return_slowpath
>                     int_ret_from_sys_call
>                     __GI___libc_write
>                     0x6f2e7473696c7665
> 
>      0.58%     0.58%  ls       [kernel.vmlinux]    [k] __enqueue_entity                 
>                  |
>                  ---__enqueue_entity
>                     pick_next_task_fair
>                     __schedule
>                     schedule
>                     prepare_exit_to_usermode
>                     syscall_return_slowpath
>                     int_ret_from_sys_call
>                     __GI___libc_write
>                     0x6f2e7473696c7665
> 
>      0.57%     0.00%  ls       libc-2.18.so        [.] __GI___munmap                    
>                  |
>                  ---__GI___munmap
> 
>      0.57%     0.00%  ls       [kernel.vmlinux]    [k] sys_munmap                       
>                  |
>                  ---sys_munmap
>                     entry_SYSCALL_64_fastpath
>                     __GI___munmap
> 
>      0.57%     0.00%  ls       [kernel.vmlinux]    [k] vm_munmap                        
>                  |
>                  ---vm_munmap
>                     sys_munmap
>                     entry_SYSCALL_64_fastpath
>                     __GI___munmap
> 
>      0.57%     0.00%  ls       [kernel.vmlinux]    [k] do_munmap                        
>                  |
>                  ---do_munmap
>                     vm_munmap
>                     sys_munmap
>                     entry_SYSCALL_64_fastpath
>                     __GI___munmap
> 
>      0.57%     0.00%  ls       [kernel.vmlinux]    [k] remove_vma                       
>                  |
>                  ---remove_vma
>                     do_munmap
>                     vm_munmap
>                     sys_munmap
>                     entry_SYSCALL_64_fastpath
>                     __GI___munmap
> 
>      0.57%     0.57%  ls       [kernel.vmlinux]    [k] kmem_cache_free                  
>                  |
>                  ---kmem_cache_free
>                     remove_vma
>                     do_munmap
>                     vm_munmap
>                     sys_munmap
>                     entry_SYSCALL_64_fastpath
>                     __GI___munmap
> 
>      0.32%     0.00%  ls       [kernel.vmlinux]    [k] msg_print_text                   
>                  |
>                  ---msg_print_text
>                     console_unlock
>                     vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>      0.32%     0.00%  ls       [kernel.vmlinux]    [k] print_prefix                     
>                  |
>                  ---print_prefix
>                     msg_print_text
>                     console_unlock
>                     vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>      0.32%     0.00%  ls       [kernel.vmlinux]    [k] print_time.part.12               
>                  |
>                  ---print_time.part.12
>                     print_prefix
>                     msg_print_text
>                     console_unlock
>                     vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>      0.32%     0.00%  ls       [kernel.vmlinux]    [k] sprintf                          
>                  |
>                  ---sprintf
>                     print_time.part.12
>                     print_prefix
>                     msg_print_text
>                     console_unlock
>                     vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>      0.32%     0.00%  ls       [kernel.vmlinux]    [k] vsnprintf                        
>                  |
>                  ---vsnprintf
>                     sprintf
>                     print_time.part.12
>                     print_prefix
>                     msg_print_text
>                     console_unlock
>                     vprintk_emit
>                     vprintk_default
>                     printk
>                     perf_duration_warn
>                     irq_work_run_list
>                     irq_work_run
>                     smp_irq_work_interrupt
>                     irq_work_interrupt
>                     __inode_permission
>                     inode_permission
>                     link_path_walk
>                     path_openat
>                     do_filp_open
>                     do_sys_open
>                     sys_open
>                     entry_SYSCALL_64_fastpath
>                     open64
>                     0x686361632e6f732e
> 
>      0.32%     0.00%  ls       ld-2.18.so          [.] 0xffff804cabb62640               
>                  |
>                  ---0x21e640
>                     0
> 
>      0.32%     0.32%  ls       ld-2.18.so          [.] _dl_setup_hash                   
>                  |
>                  ---_dl_setup_hash
>                     _dl_map_object
>                     0x21e640
>                     0
> 
>      0.29%     0.00%  ls       [unknown]           [k] 0x00007f9abb498a77               
>                  |
>                  ---0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] return_from_execve               
>                  |
>                  ---return_from_execve
>                     0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] sys_execve                       
>                  |
>                  ---sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] do_execveat_common.isra.32       
>                  |
>                  ---do_execveat_common.isra.32
>                     sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] search_binary_handler            
>                  |
>                  ---search_binary_handler
>                     do_execveat_common.isra.32
>                     sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] load_elf_binary                  
>                  |
>                  ---load_elf_binary
>                     search_binary_handler
>                     do_execveat_common.isra.32
>                     sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] setup_new_exec                   
>                  |
>                  ---setup_new_exec
>                     load_elf_binary
>                     search_binary_handler
>                     do_execveat_common.isra.32
>                     sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] apic_timer_interrupt             
>                  |
>                  ---apic_timer_interrupt
>                     setup_new_exec
>                     load_elf_binary
>                     search_binary_handler
>                     do_execveat_common.isra.32
>                     sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] smp_apic_timer_interrupt         
>                  |
>                  ---smp_apic_timer_interrupt
>                     apic_timer_interrupt
>                     setup_new_exec
>                     load_elf_binary
>                     search_binary_handler
>                     do_execveat_common.isra.32
>                     sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] local_apic_timer_interrupt       
>                  |
>                  ---local_apic_timer_interrupt
>                     smp_apic_timer_interrupt
>                     apic_timer_interrupt
>                     setup_new_exec
>                     load_elf_binary
>                     search_binary_handler
>                     do_execveat_common.isra.32
>                     sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] hrtimer_interrupt                
>                  |
>                  ---hrtimer_interrupt
>                     local_apic_timer_interrupt
>                     smp_apic_timer_interrupt
>                     apic_timer_interrupt
>                     setup_new_exec
>                     load_elf_binary
>                     search_binary_handler
>                     do_execveat_common.isra.32
>                     sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] __hrtimer_run_queues             
>                  |
>                  ---__hrtimer_run_queues
>                     hrtimer_interrupt
>                     local_apic_timer_interrupt
>                     smp_apic_timer_interrupt
>                     apic_timer_interrupt
>                     setup_new_exec
>                     load_elf_binary
>                     search_binary_handler
>                     do_execveat_common.isra.32
>                     sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] tick_sched_timer                 
>                  |
>                  ---tick_sched_timer
>                     __hrtimer_run_queues
>                     hrtimer_interrupt
>                     local_apic_timer_interrupt
>                     smp_apic_timer_interrupt
>                     apic_timer_interrupt
>                     setup_new_exec
>                     load_elf_binary
>                     search_binary_handler
>                     do_execveat_common.isra.32
>                     sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] tick_sched_handle.isra.17        
>                  |
>                  ---tick_sched_handle.isra.17
>                     tick_sched_timer
>                     __hrtimer_run_queues
>                     hrtimer_interrupt
>                     local_apic_timer_interrupt
>                     smp_apic_timer_interrupt
>                     apic_timer_interrupt
>                     setup_new_exec
>                     load_elf_binary
>                     search_binary_handler
>                     do_execveat_common.isra.32
>                     sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] update_process_times             
>                  |
>                  ---update_process_times
>                     tick_sched_handle.isra.17
>                     tick_sched_timer
>                     __hrtimer_run_queues
>                     hrtimer_interrupt
>                     local_apic_timer_interrupt
>                     smp_apic_timer_interrupt
>                     apic_timer_interrupt
>                     setup_new_exec
>                     load_elf_binary
>                     search_binary_handler
>                     do_execveat_common.isra.32
>                     sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
>      0.21%     0.00%  ls       [kernel.vmlinux]    [k] scheduler_tick                   
>                  |
>                  ---scheduler_tick
>                     update_process_times
>                     tick_sched_handle.isra.17
>                     tick_sched_timer
>                     __hrtimer_run_queues
>                     hrtimer_interrupt
>                     local_apic_timer_interrupt
>                     smp_apic_timer_interrupt
>                     apic_timer_interrupt
>                     setup_new_exec
>                     load_elf_binary
>                     search_binary_handler
>                     do_execveat_common.isra.32
>                     sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
>      0.08%     0.08%  ls       [kernel.vmlinux]    [k] native_sched_clock               
>                  |
>                  ---native_sched_clock
>                     0x7f9abb498a77
> 
>      0.00%     0.00%  ls       [kernel.vmlinux]    [k] perf_event_exec                  
>                  |
>                  ---perf_event_exec
>                     setup_new_exec
>                     load_elf_binary
>                     search_binary_handler
>                     do_execveat_common.isra.32
>                     sys_execve
>                     return_from_execve
>                     0x7f9abb498a77
> 
> 
> 
> # Samples: 158  of event 'cpu/instructions,call-graph=no,time=0,period=20000/p'
> # Event count (approx.): 3762761
> #
> # Children      Self  Command  Shared Object     Symbol                            
> # ........  ........  .......  ................  ..................................
> #
>      5.87%     5.87%  ls       [unknown]         [.] 0x00007fb353d5cf7e            
> 
>      4.84%     4.84%  ls       [kernel.vmlinux]  [k] delay_tsc                     
> 
>      3.77%     3.77%  ls       [kernel.vmlinux]  [k] pick_next_task_fair           
> 
>      3.11%     3.11%  ls       [kernel.vmlinux]  [k] unmap_page_range              
> 
>      2.94%     2.94%  ls       [kernel.vmlinux]  [k] n_tty_write                   
> 
>      2.48%     2.48%  ls       [unknown]         [.] 0x000000000040669a            
> 
>      2.40%     2.40%  ls       [unknown]         [.] 0x00007fb353d5cd1a            
> 
>      2.37%     2.37%  ls       [unknown]         [.] 0x00007fb353d5cca4            
> 
>      2.37%     2.37%  ls       [unknown]         [.] 0x00000000004066ca            
> 
>      2.25%     2.25%  ls       [unknown]         [.] 0x00007fb353d5db87            
> 
>      2.03%     2.03%  ls       [unknown]         [.] 0x00007fb353d5ccaa            
> 
>      1.98%     1.98%  ls       [unknown]         [.] 0x00007fb353d5cc9c            
> 
>      1.97%     1.97%  ls       [unknown]         [.] 0x0000000000405b2a            
> 
>      1.89%     1.89%  ls       [unknown]         [.] 0x00007fb3546d43e0            
> 
>      1.83%     1.83%  ls       [unknown]         [.] 0x00007fb353d5cef6            
> 
>      1.75%     1.75%  ls       [kernel.vmlinux]  [k] ring_buffer_record_is_on      
> 
>      1.69%     1.69%  ls       [unknown]         [.] 0x00007fb353d5cc3a            
> 
>      1.63%     1.63%  ls       [unknown]         [.] 0x0000000000405611            
> 
>      1.47%     1.47%  ls       [kernel.vmlinux]  [k] __fput                        
> 
>      1.44%     1.44%  ls       [kernel.vmlinux]  [k] release_pages                 
> 
>      1.37%     1.37%  ls       [kernel.vmlinux]  [k] perf_pmu_sched_task           
> 
>      1.34%     1.34%  ls       [unknown]         [.] 0x00007fb353d54fed            
> 
>      1.33%     1.33%  ls       [kernel.vmlinux]  [k] __memcpy                      
> 
>      1.33%     1.33%  ls       [unknown]         [.] 0x00007fb353d5df34            
> 
>      1.31%     1.31%  ls       [unknown]         [.] 0x00007fb353d5ccdf            
> 
>      1.29%     1.29%  ls       [unknown]         [.] 0x0000000000406a19            
> 
>      1.29%     1.29%  ls       [kernel.vmlinux]  [k] page_remove_rmap              
> 
>      1.29%     1.29%  ls       [kernel.vmlinux]  [k] do_set_pte                    
> 
>      1.28%     1.28%  ls       [kernel.vmlinux]  [k] __audit_syscall_exit          
> 
>      1.28%     1.28%  ls       [kernel.vmlinux]  [k] enqueue_entity                
> 
>      1.28%     1.28%  ls       [ext4]            [k] ext4fs_dirhash                
> 
>      1.26%     1.26%  ls       [kernel.vmlinux]  [k] cpuacct_charge                
> 
>      1.24%     1.24%  ls       [kernel.vmlinux]  [k] sched_clock                   
> 
>      1.24%     1.24%  ls       [unknown]         [.] 0x00007fb353d4bca5            
> 
>      1.22%     1.22%  ls       [unknown]         [.] 0x00007fb353dfb6ac            
> 
>      1.21%     1.21%  ls       [ext4]            [k] str2hashbuf_signed            
> 
>      1.21%     1.21%  ls       [unknown]         [.] 0x00007fb3546c766a            
> 
>      1.20%     1.20%  ls       [ext4]            [k] __ext4_check_dir_entry        
> 
>      1.13%     1.13%  ls       [kernel.vmlinux]  [k] x86_pmu_disable               
> 
>      1.12%     1.12%  ls       [unknown]         [.] 0x00007fb3546c5571            
> 
>      1.11%     1.11%  ls       [kernel.vmlinux]  [k] vm_normal_page                
> 
>      1.09%     1.09%  ls       [unknown]         [.] 0x00007fb3546cc246            
> 
>      1.06%     1.06%  ls       [unknown]         [.] 0x00007fb3546cc21e            
> 
>      1.03%     1.03%  ls       [unknown]         [.] 0x000000000040e6a0            
> 
>      1.01%     1.01%  ls       [unknown]         [.] 0x00007fb3546c5ce3            
> 
>      0.88%     0.88%  ls       [unknown]         [.] 0x00007fb3546c7814            
> 
>      0.88%     0.88%  ls       [kernel.vmlinux]  [k] mark_page_accessed            
> 
>      0.87%     0.87%  ls       [unknown]         [.] 0x00007fb3546c7960            
> 
>      0.78%     0.78%  ls       [unknown]         [.] 0x000000000040e7ea            
> 
>      0.77%     0.77%  ls       [kernel.vmlinux]  [k] security_mmap_file            
> 
>      0.66%     0.66%  ls       [unknown]         [.] 0x000000000040f3c5            
> 
>      0.65%     0.65%  ls       [kernel.vmlinux]  [k] enqueue_task_fair             
> 
>      0.63%     0.63%  ls       [kernel.vmlinux]  [k] unlock_page                   
> 
>      0.61%     0.61%  ls       [kernel.vmlinux]  [k] place_entity                  
> 
>      0.57%     0.57%  ls       [unknown]         [.] 0x000000000040e808            
> 
>      0.56%     0.56%  ls       [kernel.vmlinux]  [k] perf_event_aux_ctx            
> 
>      0.54%     0.54%  ls       [kernel.vmlinux]  [k] __rb_insert_augmented         
> 
>      0.54%     0.54%  ls       [kernel.vmlinux]  [k] update_min_vruntime           
> 
>      0.50%     0.50%  ls       [kernel.vmlinux]  [k] update_curr                   
> 
>      0.47%     0.47%  ls       [kernel.vmlinux]  [k] vma_interval_tree_insert      
> 
>      0.47%     0.47%  ls       [kernel.vmlinux]  [k] io_serial_in                  
> 
>      0.47%     0.47%  ls       [kernel.vmlinux]  [k] check_cfs_rq_runtime          
> 
>      0.46%     0.46%  ls       [kernel.vmlinux]  [k] tracing_record_cmdline        
> 
>      0.46%     0.46%  ls       [kernel.vmlinux]  [k] __dec_zone_page_state         
> 
>      0.42%     0.42%  ls       [kernel.vmlinux]  [k] try_to_wake_up                
> 
>      0.40%     0.40%  ls       [kernel.vmlinux]  [k] insert_work                   
> 
>      0.40%     0.40%  ls       [kernel.vmlinux]  [k] put_prev_entity               
> 
>      0.40%     0.40%  ls       [kernel.vmlinux]  [k] avc_has_perm_noaudit          
> 
>      0.38%     0.38%  ls       [kernel.vmlinux]  [k] __list_del_entry              
> 
>      0.38%     0.38%  ls       [unknown]         [.] 0x0000000000405852            
> 
>      0.36%     0.36%  ls       [kernel.vmlinux]  [k] __intel_pmu_disable_all       
> 
>      0.35%     0.35%  ls       [kernel.vmlinux]  [k] ttwu_do_activate.constprop.92 
> 
>      0.35%     0.35%  ls       [kernel.vmlinux]  [k] mem_cgroup_begin_page_stat    
> 
>      0.34%     0.34%  ls       [kernel.vmlinux]  [k] perf_ctx_unlock               
> 
>      0.32%     0.32%  ls       [unknown]         [.] 0x0000000000405b46            
> 
>      0.32%     0.32%  ls       [unknown]         [.] 0x0000000000405811            
> 
>      0.31%     0.31%  ls       [unknown]         [.] 0x00007fb3546c54fa            
> 
>      0.31%     0.31%  ls       [kernel.vmlinux]  [k] rb_next                       
> 
>      0.31%     0.31%  ls       [kernel.vmlinux]  [k] tracing_is_on                 
> 
>      0.30%     0.30%  ls       [unknown]         [.] 0x00007fb3546c553d            
> 
>      0.29%     0.29%  ls       [unknown]         [.] 0x00007fb3546cb597            
> 
>      0.20%     0.20%  ls       [kernel.vmlinux]  [k] format_decode                 
> 
>      0.18%     0.18%  ls       [kernel.vmlinux]  [k] __audit_syscall_entry         
> 
>      0.16%     0.16%  ls       [kernel.vmlinux]  [k] arch_get_unmapped_area_topdown
> 
>      0.16%     0.16%  ls       [kernel.vmlinux]  [k] fsnotify                      
> 
>      0.16%     0.16%  ls       [kernel.vmlinux]  [k] intel_bts_enable_local        
> 
>      0.15%     0.15%  ls       [kernel.vmlinux]  [k] selinux_inode_permission      
> 
>      0.14%     0.14%  ls       [kernel.vmlinux]  [k] wait_for_xmitr                
> 
>      0.14%     0.14%  ls       [kernel.vmlinux]  [k] __const_udelay                
> 
>      0.11%     0.11%  ls       [kernel.vmlinux]  [k] find_vma                      
> 
>      0.11%     0.11%  ls       [unknown]         [.] 0x00007fb3546d18ae            
> 
>      0.10%     0.10%  ls       [unknown]         [.] 0x00007fb3546cb18e            
> 
>      0.03%     0.03%  ls       [kernel.vmlinux]  [k] native_sched_clock            
> 
>      0.01%     0.01%  ls       [kernel.vmlinux]  [k] trigger_load_balance          
> 
>      0.01%     0.01%  ls       [kernel.vmlinux]  [k] do_nmi                        
> 
>      0.00%     0.00%  ls       [kernel.vmlinux]  [k] perf_event_nmi_handler        
> 
>      0.00%     0.00%  ls       [kernel.vmlinux]  [k] perf_sample_event_took        
> 
>      0.00%     0.00%  ls       [kernel.vmlinux]  [k] nmi                           
> 
> 
> 
> #
> # (For a higher level overview, try: perf report --sort comm,dso)
> #


^ permalink raw reply	[flat|nested] 20+ messages in thread

* Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support
  2015-08-10 12:56       ` Liang, Kan
@ 2015-08-10 15:39         ` Arnaldo Carvalho de Melo
  2015-08-10 18:57           ` Liang, Kan
  0 siblings, 1 reply; 20+ messages in thread
From: Arnaldo Carvalho de Melo @ 2015-08-10 15:39 UTC (permalink / raw)
  To: Liang, Kan; +Cc: Jiri Olsa, Namhyung Kim, Andi Kleen, linux-kernel

Em Mon, Aug 10, 2015 at 12:56:04PM +0000, Liang, Kan escreveu:
> > Em Fri, Aug 07, 2015 at 12:38:43PM -0300, Arnaldo Carvalho de Melo escreveu:
> > [root@zoo ~]# perf report --header-only
> > # cmdline :
> > /home/acme/bin/perf record -e {cpu/cpu-cycles,call-
> > graph=fp,time,period=10000/pp,cpu/instructions,call-
> > graph=no,time=0,period=20000/p} -a
<SNIP>

> > I get:

> > Samples: 2K of event 'cpu/instructions,call-
> > graph=no,time=0,period=20000/p', Event count (approx.): 46956518
> >   Children      Self  Command          Shared Object Symbol                  ◆
> > -   67.56%     0.00%  qemu-system-x86  [unknown]     [.]
> > 0xad5e258d4c544155  ▒
> >      0xad5e258d4c544155                                                      ▒
> > -   67.56%     0.00%  qemu-system-x86  libc-2.20.so  [.] __libc_start_main   ▒
> >      __libc_start_main                                                       ▒
> >      0xad5e258d4c544155                                                      ▒

> > This is in the 'perf report' TUI, why, for an event with 'callgraph=no', we get
> > callchains? How come?
 
> That's the design.
> For sampling multiple events, it may not be needed to collect callgraphs for
> all of them. Because the sample sites are usually nearby. It's enough to collect
> the callgraphs on a reference event. For other events, it can still show callgraphs
> according to the callgraphs on a reference event.

So, "call-graph=no" doesn't mean you don't want callchains for a
particular events _if_ there is another event in the group for which
callchains is available.

But if "call-graph=no" for all events, then, yes, "no" means really
"no". :-)

I think we should use "call-graph=ref" to mean that no callchains should
be requested to the kernel infrastructure for that particular event, but
that when doing the report, use callchains available in some other event
(perhaps would be good to specify which one), while "call-graph=no"
really means "no", i.e. no callchains asked from the kernel for this
event, and _no_ callchains to appear on report.

If "ref" is used and no callchains are available anywhere, that is a bug
as well, i.e. I asked for callchains up to a event to be used, by
getting that info from another event, but no event has callchains:
error.

- Arnaldo


^ permalink raw reply	[flat|nested] 20+ messages in thread

* RE: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support
  2015-08-10 15:39         ` Arnaldo Carvalho de Melo
@ 2015-08-10 18:57           ` Liang, Kan
  2015-08-10 19:39             ` Arnaldo Carvalho de Melo
  0 siblings, 1 reply; 20+ messages in thread
From: Liang, Kan @ 2015-08-10 18:57 UTC (permalink / raw)
  To: Arnaldo Carvalho de Melo
  Cc: Jiri Olsa, Namhyung Kim, Andi Kleen, linux-kernel

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset="utf-8", Size: 2638 bytes --]



> <SNIP>
> 
> > > I get:
> 
> > > Samples: 2K of event 'cpu/instructions,call-
> > > graph=no,time=0,period=20000/p', Event count (approx.): 46956518
> > >   Children      Self  Command          Shared Object Symbol                  â—†
> > > -   67.56%     0.00%  qemu-system-x86  [unknown]     [.]
> > > 0xad5e258d4c544155  â–’
> > >      0xad5e258d4c544155                                                      â–’
> > > -   67.56%     0.00%  qemu-system-x86  libc-2.20.so  [.] __libc_start_main
> â–’
> > >      __libc_start_main                                                       â–’
> > >      0xad5e258d4c544155                                                      â–’
> 
> > > This is in the 'perf report' TUI, why, for an event with
> > > 'callgraph=no', we get callchains? How come?
> 
> > That's the design.
> > For sampling multiple events, it may not be needed to collect
> > callgraphs for all of them. Because the sample sites are usually
> > nearby. It's enough to collect the callgraphs on a reference event.
> > For other events, it can still show callgraphs according to the callgraphs on
> a reference event.
> 
> So, "call-graph=no" doesn't mean you don't want callchains for a particular
> events _if_ there is another event in the group for which callchains is
> available.
> 
> But if "call-graph=no" for all events, then, yes, "no" means really "no". :-)
> 
> I think we should use "call-graph=ref" to mean that no callchains should be
> requested to the kernel infrastructure for that particular event, but that
> when doing the report, use callchains available in some other event
> (perhaps would be good to specify which one), while "call-graph=no"
> really means "no", i.e. no callchains asked from the kernel for this event,
> and _no_ callchains to appear on report.
> 
> If "ref" is used and no callchains are available anywhere, that is a bug as
> well, i.e. I asked for callchains up to a event to be used, by getting that info
> from another event, but no event has callchains:
> error.
> 

If we use " call-graph=ref", it means "ref" is a new callchain mode. But it's not.
I think the "ref" thing should only impact the perf report.
So we may introduce a new option "--show-callchain-ref" for that purpose.
If it applied, the available callchain information from other event will be
printed for "call-graph=no" event.
If not, no callchain information is printed for "call-graph=no" event.
The default is no print.
Is it OK?

Thanks,
Kan
ÿôèº{.nÇ+‰·Ÿ®‰­†+%ŠËÿ±éݶ\x17¥Šwÿº{.nÇ+‰·¥Š{±þG«éÿŠ{ayº\x1dʇڙë,j\a­¢f£¢·hšïêÿ‘êçz_è®\x03(­éšŽŠÝ¢j"ú\x1a¶^[m§ÿÿ¾\a«þG«éÿ¢¸?™¨è­Ú&£ø§~á¶iO•æ¬z·švØ^\x14\x04\x1a¶^[m§ÿÿÃ\fÿ¶ìÿ¢¸?–I¥

^ permalink raw reply	[flat|nested] 20+ messages in thread

* Re: [PATCH RFC V9 2/3] perf,tools: per-event callgraph support
  2015-08-10 18:57           ` Liang, Kan
@ 2015-08-10 19:39             ` Arnaldo Carvalho de Melo
  0 siblings, 0 replies; 20+ messages in thread
From: Arnaldo Carvalho de Melo @ 2015-08-10 19:39 UTC (permalink / raw)
  To: Liang, Kan; +Cc: Jiri Olsa, Namhyung Kim, Andi Kleen, linux-kernel

Em Mon, Aug 10, 2015 at 06:57:20PM +0000, Liang, Kan escreveu:
> > <SNIP>

> > > > I get:

> > > > Samples: 2K of event 'cpu/instructions,call-
> > > > graph=no,time=0,period=20000/p', Event count (approx.): 46956518
> > > >   Children      Self  Command          Shared Object Symbol                  ◆
> > > > -   67.56%     0.00%  qemu-system-x86  [unknown]     [.]
> > > > 0xad5e258d4c544155  ▒
> > > >      0xad5e258d4c544155                                                      ▒
> > > > -   67.56%     0.00%  qemu-system-x86  libc-2.20.so  [.] __libc_start_main
> > > >      __libc_start_main                                                       ▒
> > > >      0xad5e258d4c544155                                                      ▒

> > > > This is in the 'perf report' TUI, why, for an event with
> > > > 'callgraph=no', we get callchains? How come?

> > > That's the design.
> > > For sampling multiple events, it may not be needed to collect
> > > callgraphs for all of them. Because the sample sites are usually
> > > nearby. It's enough to collect the callgraphs on a reference event.
> > > For other events, it can still show callgraphs according to the callgraphs on
> > a reference event.

> > So, "call-graph=no" doesn't mean you don't want callchains for a particular
> > events _if_ there is another event in the group for which callchains is
> > available.

> > But if "call-graph=no" for all events, then, yes, "no" means really "no". :-)

> > I think we should use "call-graph=ref" to mean that no callchains should be
> > requested to the kernel infrastructure for that particular event, but that
> > when doing the report, use callchains available in some other event
> > (perhaps would be good to specify which one), while "call-graph=no"
> > really means "no", i.e. no callchains asked from the kernel for this event,
> > and _no_ callchains to appear on report.

> > If "ref" is used and no callchains are available anywhere, that is a bug as
> > well, i.e. I asked for callchains up to a event to be used, by getting that info
> > from another event, but no event has callchains:
> > error.
 
> If we use " call-graph=ref", it means "ref" is a new callchain mode. But it's not.
> I think the "ref" thing should only impact the perf report.

I don't have much of a problem with that, but using "ref" to make the
intention, i.e. use reference callchains, documented, clear, makes sense to me.

I.e. when you ask for two events, one with callchains and the other without it
doesn't necessarily means we want callchains appearing on the ones we have not
enabled them.

> So we may introduce a new option "--show-callchain-ref" for that purpose.
> If it applied, the available callchain information from other event will be
> printed for "call-graph=no" event.

Ok, if the user explicitely asked for "--show-callchain-ref", then
he/she will not get confused seeing callchains for an event with
"call-graph=no".

Ah, probably --show-ref-call-graph should be better, to keep it consistent with
all the other options dealing with call-graph stuff.

> If not, no callchain information is printed for "call-graph=no" event.
> The default is no print.

Agreed, I think this almost completely reduces the possible source of
confusion.

> Is it OK?

Ok.

One possible improvement to your proposal: When showing callchains in
reference mode, make that extra explicit by adding some marker on the
side of the event name.

I.e. right now we will see callchains, when this is with another event with
callchains:

  Samples: 24  of event 'cpu/instructions,call-graph=no,time=0,period=20000/p', Event count (approx.): 480000
  Overhead  Command  Shared Object     Symbol
    12.50%  usleep   libc-2.20.so      [.] _dl_addr

My suggestion is to have something like:

  Samples: 24  of event 'cpu/instructions,call-graph=no,time=0,period=20000/p', ref cg, Event count (approx.): 480000
  Overhead  Command  Shared Object     Symbol
    12.50%  usleep   libc-2.20.so      [.] _dl_addr

See that ", ref cg"?

But would be just to remove the confusion of seeing, on the same screen,
"call-graph=no" when one _sees_ call graphs.

- Arnaldo

^ permalink raw reply	[flat|nested] 20+ messages in thread

* [tip:perf/core] perf stat: Move perf_counts struct and functions into separate object
  2015-08-07 10:51 ` [PATCH] perf tools: Move perf_counts struct and functions into separate object Jiri Olsa
  2015-08-07 13:14   ` Arnaldo Carvalho de Melo
@ 2015-08-12 12:28   ` tip-bot for Jiri Olsa
  1 sibling, 0 replies; 20+ messages in thread
From: tip-bot for Jiri Olsa @ 2015-08-12 12:28 UTC (permalink / raw)
  To: linux-tip-commits
  Cc: ak, mingo, hpa, namhyung, kan.liang, acme, tglx, jolsa,
	linux-kernel, jolsa

Commit-ID:  d809560b36a7ed31fbaf3719fdf79ddcbd30950b
Gitweb:     http://git.kernel.org/tip/d809560b36a7ed31fbaf3719fdf79ddcbd30950b
Author:     Jiri Olsa <jolsa@redhat.com>
AuthorDate: Fri, 7 Aug 2015 12:51:03 +0200
Committer:  Arnaldo Carvalho de Melo <acme@redhat.com>
CommitDate: Sat, 8 Aug 2015 14:16:49 -0300

perf stat: Move perf_counts struct and functions into separate object

Moving 'struct perf_counts' and associated functions into separate
object, so we could remove stat.c object dependency from python build.

It makes the python code to build properly, because it fails to load due
to missing stat-shadow.c object dependency if some patches from Kan
Liang are applied.

So apply this one, then Kan's.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Jiri Olsa <jolsa@kernel.org>
Cc: Kan Liang <kan.liang@intel.com>
Cc: Namhyung Kim <namhyung@kernel.org>
Link: http://lkml.kernel.org/r/20150807105103.GB8624@krava.brq.redhat.com
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
---
 tools/perf/builtin-stat.c          |  1 +
 tools/perf/util/Build              |  1 +
 tools/perf/util/counts.c           | 52 ++++++++++++++++++++++++++++++++++++++
 tools/perf/util/counts.h           | 37 +++++++++++++++++++++++++++
 tools/perf/util/evsel.h            |  2 +-
 tools/perf/util/python-ext-sources |  2 +-
 tools/perf/util/stat.c             | 49 -----------------------------------
 tools/perf/util/stat.h             | 30 ----------------------
 8 files changed, 93 insertions(+), 81 deletions(-)

diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c
index a054ddc..7aa039b 100644
--- a/tools/perf/builtin-stat.c
+++ b/tools/perf/builtin-stat.c
@@ -58,6 +58,7 @@
 #include "util/cpumap.h"
 #include "util/thread.h"
 #include "util/thread_map.h"
+#include "util/counts.h"
 
 #include <stdlib.h>
 #include <sys/prctl.h>
diff --git a/tools/perf/util/Build b/tools/perf/util/Build
index 2ee81d7..1ce0adc 100644
--- a/tools/perf/util/Build
+++ b/tools/perf/util/Build
@@ -68,6 +68,7 @@ libperf-y += target.o
 libperf-y += rblist.o
 libperf-y += intlist.o
 libperf-y += vdso.o
+libperf-y += counts.o
 libperf-y += stat.o
 libperf-y += stat-shadow.o
 libperf-y += record.o
diff --git a/tools/perf/util/counts.c b/tools/perf/util/counts.c
new file mode 100644
index 0000000..e3fde31
--- /dev/null
+++ b/tools/perf/util/counts.c
@@ -0,0 +1,52 @@
+#include <stdlib.h>
+#include "evsel.h"
+#include "counts.h"
+
+struct perf_counts *perf_counts__new(int ncpus, int nthreads)
+{
+	struct perf_counts *counts = zalloc(sizeof(*counts));
+
+	if (counts) {
+		struct xyarray *values;
+
+		values = xyarray__new(ncpus, nthreads, sizeof(struct perf_counts_values));
+		if (!values) {
+			free(counts);
+			return NULL;
+		}
+
+		counts->values = values;
+	}
+
+	return counts;
+}
+
+void perf_counts__delete(struct perf_counts *counts)
+{
+	if (counts) {
+		xyarray__delete(counts->values);
+		free(counts);
+	}
+}
+
+static void perf_counts__reset(struct perf_counts *counts)
+{
+	xyarray__reset(counts->values);
+}
+
+void perf_evsel__reset_counts(struct perf_evsel *evsel)
+{
+	perf_counts__reset(evsel->counts);
+}
+
+int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads)
+{
+	evsel->counts = perf_counts__new(ncpus, nthreads);
+	return evsel->counts != NULL ? 0 : -ENOMEM;
+}
+
+void perf_evsel__free_counts(struct perf_evsel *evsel)
+{
+	perf_counts__delete(evsel->counts);
+	evsel->counts = NULL;
+}
diff --git a/tools/perf/util/counts.h b/tools/perf/util/counts.h
new file mode 100644
index 0000000..34d8baa
--- /dev/null
+++ b/tools/perf/util/counts.h
@@ -0,0 +1,37 @@
+#ifndef __PERF_COUNTS_H
+#define __PERF_COUNTS_H
+
+#include "xyarray.h"
+
+struct perf_counts_values {
+	union {
+		struct {
+			u64 val;
+			u64 ena;
+			u64 run;
+		};
+		u64 values[3];
+	};
+};
+
+struct perf_counts {
+	s8			  scaled;
+	struct perf_counts_values aggr;
+	struct xyarray		  *values;
+};
+
+
+static inline struct perf_counts_values*
+perf_counts(struct perf_counts *counts, int cpu, int thread)
+{
+	return xyarray__entry(counts->values, cpu, thread);
+}
+
+struct perf_counts *perf_counts__new(int ncpus, int nthreads);
+void perf_counts__delete(struct perf_counts *counts);
+
+void perf_evsel__reset_counts(struct perf_evsel *evsel);
+int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads);
+void perf_evsel__free_counts(struct perf_evsel *evsel);
+
+#endif /* __PERF_COUNTS_H */
diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h
index 6a12908..b948f69 100644
--- a/tools/perf/util/evsel.h
+++ b/tools/perf/util/evsel.h
@@ -9,7 +9,7 @@
 #include "xyarray.h"
 #include "symbol.h"
 #include "cpumap.h"
-#include "stat.h"
+#include "counts.h"
 
 struct perf_evsel;
 
diff --git a/tools/perf/util/python-ext-sources b/tools/perf/util/python-ext-sources
index 0766d98..51be28b 100644
--- a/tools/perf/util/python-ext-sources
+++ b/tools/perf/util/python-ext-sources
@@ -16,7 +16,7 @@ util/util.c
 util/xyarray.c
 util/cgroup.c
 util/rblist.c
-util/stat.c
+util/counts.c
 util/strlist.c
 util/trace-event.c
 ../lib/rbtree.c
diff --git a/tools/perf/util/stat.c b/tools/perf/util/stat.c
index c5c709c..415c359 100644
--- a/tools/perf/util/stat.c
+++ b/tools/perf/util/stat.c
@@ -97,55 +97,6 @@ void perf_stat_evsel_id_init(struct perf_evsel *evsel)
 	}
 }
 
-struct perf_counts *perf_counts__new(int ncpus, int nthreads)
-{
-	struct perf_counts *counts = zalloc(sizeof(*counts));
-
-	if (counts) {
-		struct xyarray *values;
-
-		values = xyarray__new(ncpus, nthreads, sizeof(struct perf_counts_values));
-		if (!values) {
-			free(counts);
-			return NULL;
-		}
-
-		counts->values = values;
-	}
-
-	return counts;
-}
-
-void perf_counts__delete(struct perf_counts *counts)
-{
-	if (counts) {
-		xyarray__delete(counts->values);
-		free(counts);
-	}
-}
-
-static void perf_counts__reset(struct perf_counts *counts)
-{
-	xyarray__reset(counts->values);
-}
-
-void perf_evsel__reset_counts(struct perf_evsel *evsel)
-{
-	perf_counts__reset(evsel->counts);
-}
-
-int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads)
-{
-	evsel->counts = perf_counts__new(ncpus, nthreads);
-	return evsel->counts != NULL ? 0 : -ENOMEM;
-}
-
-void perf_evsel__free_counts(struct perf_evsel *evsel)
-{
-	perf_counts__delete(evsel->counts);
-	evsel->counts = NULL;
-}
-
 void perf_evsel__reset_stat_priv(struct perf_evsel *evsel)
 {
 	int i;
diff --git a/tools/perf/util/stat.h b/tools/perf/util/stat.h
index 0b897b0..62448c8 100644
--- a/tools/perf/util/stat.h
+++ b/tools/perf/util/stat.h
@@ -33,23 +33,6 @@ enum aggr_mode {
 	AGGR_THREAD,
 };
 
-struct perf_counts_values {
-	union {
-		struct {
-			u64 val;
-			u64 ena;
-			u64 run;
-		};
-		u64 values[3];
-	};
-};
-
-struct perf_counts {
-	s8			  scaled;
-	struct perf_counts_values aggr;
-	struct xyarray		  *values;
-};
-
 struct perf_stat_config {
 	enum aggr_mode	aggr_mode;
 	bool		scale;
@@ -57,12 +40,6 @@ struct perf_stat_config {
 	unsigned int	interval;
 };
 
-static inline struct perf_counts_values*
-perf_counts(struct perf_counts *counts, int cpu, int thread)
-{
-	return xyarray__entry(counts->values, cpu, thread);
-}
-
 void update_stats(struct stats *stats, u64 val);
 double avg_stats(struct stats *stats);
 double stddev_stats(struct stats *stats);
@@ -96,13 +73,6 @@ void perf_stat__update_shadow_stats(struct perf_evsel *counter, u64 *count,
 void perf_stat__print_shadow_stats(FILE *out, struct perf_evsel *evsel,
 				   double avg, int cpu, enum aggr_mode aggr);
 
-struct perf_counts *perf_counts__new(int ncpus, int nthreads);
-void perf_counts__delete(struct perf_counts *counts);
-
-void perf_evsel__reset_counts(struct perf_evsel *evsel);
-int perf_evsel__alloc_counts(struct perf_evsel *evsel, int ncpus, int nthreads);
-void perf_evsel__free_counts(struct perf_evsel *evsel);
-
 void perf_evsel__reset_stat_priv(struct perf_evsel *evsel);
 int perf_evsel__alloc_stat_priv(struct perf_evsel *evsel);
 void perf_evsel__free_stat_priv(struct perf_evsel *evsel);

^ permalink raw reply related	[flat|nested] 20+ messages in thread

* [tip:perf/core] perf callchain: Move option parsing code to util.c
  2015-08-06 19:44 [PATCH RFC V9 1/3] perf,tools: move callchain option parse code to util.c kan.liang
                   ` (2 preceding siblings ...)
  2015-08-07 10:51 ` [PATCH] perf tools: Move perf_counts struct and functions into separate object Jiri Olsa
@ 2015-08-12 12:29 ` tip-bot for Kan Liang
  3 siblings, 0 replies; 20+ messages in thread
From: tip-bot for Kan Liang @ 2015-08-12 12:29 UTC (permalink / raw)
  To: linux-tip-commits
  Cc: acme, kan.liang, jolsa, namhyung, mingo, linux-kernel, hpa, ak, tglx

Commit-ID:  076a30c411ba2b91a18d44a5a01977035cdd7716
Gitweb:     http://git.kernel.org/tip/076a30c411ba2b91a18d44a5a01977035cdd7716
Author:     Kan Liang <kan.liang@intel.com>
AuthorDate: Thu, 6 Aug 2015 15:44:52 -0400
Committer:  Arnaldo Carvalho de Melo <acme@redhat.com>
CommitDate: Sat, 8 Aug 2015 14:16:49 -0300

perf callchain: Move option parsing code to util.c

Move callchain option parse related code to util.c, to avoid dragging
more object files into the python binding.

Signed-off-by: Kan Liang <kan.liang@intel.com>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Jiri Olsa <jolsa@kernel.org>
Cc: Namhyung Kim <namhyung@kernel.org>
Link: http://lkml.kernel.org/r/1438890294-33409-1-git-send-email-kan.liang@intel.com
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
---
 tools/perf/util/callchain.c | 89 +-------------------------------------------
 tools/perf/util/callchain.h |  1 +
 tools/perf/util/util.c      | 90 +++++++++++++++++++++++++++++++++++++++++++++
 tools/perf/util/util.h      |  2 +
 4 files changed, 94 insertions(+), 88 deletions(-)

diff --git a/tools/perf/util/callchain.c b/tools/perf/util/callchain.c
index 931cca8..773fe13 100644
--- a/tools/perf/util/callchain.c
+++ b/tools/perf/util/callchain.c
@@ -25,96 +25,9 @@
 
 __thread struct callchain_cursor callchain_cursor;
 
-#ifdef HAVE_DWARF_UNWIND_SUPPORT
-static int get_stack_size(const char *str, unsigned long *_size)
-{
-	char *endptr;
-	unsigned long size;
-	unsigned long max_size = round_down(USHRT_MAX, sizeof(u64));
-
-	size = strtoul(str, &endptr, 0);
-
-	do {
-		if (*endptr)
-			break;
-
-		size = round_up(size, sizeof(u64));
-		if (!size || size > max_size)
-			break;
-
-		*_size = size;
-		return 0;
-
-	} while (0);
-
-	pr_err("callchain: Incorrect stack dump size (max %ld): %s\n",
-	       max_size, str);
-	return -1;
-}
-#endif /* HAVE_DWARF_UNWIND_SUPPORT */
-
 int parse_callchain_record_opt(const char *arg, struct callchain_param *param)
 {
-	char *tok, *name, *saveptr = NULL;
-	char *buf;
-	int ret = -1;
-
-	/* We need buffer that we know we can write to. */
-	buf = malloc(strlen(arg) + 1);
-	if (!buf)
-		return -ENOMEM;
-
-	strcpy(buf, arg);
-
-	tok = strtok_r((char *)buf, ",", &saveptr);
-	name = tok ? : (char *)buf;
-
-	do {
-		/* Framepointer style */
-		if (!strncmp(name, "fp", sizeof("fp"))) {
-			if (!strtok_r(NULL, ",", &saveptr)) {
-				param->record_mode = CALLCHAIN_FP;
-				ret = 0;
-			} else
-				pr_err("callchain: No more arguments "
-				       "needed for --call-graph fp\n");
-			break;
-
-#ifdef HAVE_DWARF_UNWIND_SUPPORT
-		/* Dwarf style */
-		} else if (!strncmp(name, "dwarf", sizeof("dwarf"))) {
-			const unsigned long default_stack_dump_size = 8192;
-
-			ret = 0;
-			param->record_mode = CALLCHAIN_DWARF;
-			param->dump_size = default_stack_dump_size;
-
-			tok = strtok_r(NULL, ",", &saveptr);
-			if (tok) {
-				unsigned long size = 0;
-
-				ret = get_stack_size(tok, &size);
-				param->dump_size = size;
-			}
-#endif /* HAVE_DWARF_UNWIND_SUPPORT */
-		} else if (!strncmp(name, "lbr", sizeof("lbr"))) {
-			if (!strtok_r(NULL, ",", &saveptr)) {
-				param->record_mode = CALLCHAIN_LBR;
-				ret = 0;
-			} else
-				pr_err("callchain: No more arguments "
-					"needed for --call-graph lbr\n");
-			break;
-		} else {
-			pr_err("callchain: Unknown --call-graph option "
-			       "value: %s\n", arg);
-			break;
-		}
-
-	} while (0);
-
-	free(buf);
-	return ret;
+	return parse_callchain_record(arg, param);
 }
 
 static int parse_callchain_mode(const char *value)
diff --git a/tools/perf/util/callchain.h b/tools/perf/util/callchain.h
index 68a32c2..acee2b3 100644
--- a/tools/perf/util/callchain.h
+++ b/tools/perf/util/callchain.h
@@ -177,6 +177,7 @@ int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *
 			bool hide_unresolved);
 
 extern const char record_callchain_help[];
+extern int parse_callchain_record(const char *arg, struct callchain_param *param);
 int parse_callchain_record_opt(const char *arg, struct callchain_param *param);
 int parse_callchain_report_opt(const char *arg);
 int perf_callchain_config(const char *var, const char *value);
diff --git a/tools/perf/util/util.c b/tools/perf/util/util.c
index edc2d63..f7adf12 100644
--- a/tools/perf/util/util.c
+++ b/tools/perf/util/util.c
@@ -566,6 +566,96 @@ unsigned long parse_tag_value(const char *str, struct parse_tag *tags)
 	return (unsigned long) -1;
 }
 
+int get_stack_size(const char *str, unsigned long *_size)
+{
+	char *endptr;
+	unsigned long size;
+	unsigned long max_size = round_down(USHRT_MAX, sizeof(u64));
+
+	size = strtoul(str, &endptr, 0);
+
+	do {
+		if (*endptr)
+			break;
+
+		size = round_up(size, sizeof(u64));
+		if (!size || size > max_size)
+			break;
+
+		*_size = size;
+		return 0;
+
+	} while (0);
+
+	pr_err("callchain: Incorrect stack dump size (max %ld): %s\n",
+	       max_size, str);
+	return -1;
+}
+
+int parse_callchain_record(const char *arg, struct callchain_param *param)
+{
+	char *tok, *name, *saveptr = NULL;
+	char *buf;
+	int ret = -1;
+
+	/* We need buffer that we know we can write to. */
+	buf = malloc(strlen(arg) + 1);
+	if (!buf)
+		return -ENOMEM;
+
+	strcpy(buf, arg);
+
+	tok = strtok_r((char *)buf, ",", &saveptr);
+	name = tok ? : (char *)buf;
+
+	do {
+		/* Framepointer style */
+		if (!strncmp(name, "fp", sizeof("fp"))) {
+			if (!strtok_r(NULL, ",", &saveptr)) {
+				param->record_mode = CALLCHAIN_FP;
+				ret = 0;
+			} else
+				pr_err("callchain: No more arguments "
+				       "needed for --call-graph fp\n");
+			break;
+
+#ifdef HAVE_DWARF_UNWIND_SUPPORT
+		/* Dwarf style */
+		} else if (!strncmp(name, "dwarf", sizeof("dwarf"))) {
+			const unsigned long default_stack_dump_size = 8192;
+
+			ret = 0;
+			param->record_mode = CALLCHAIN_DWARF;
+			param->dump_size = default_stack_dump_size;
+
+			tok = strtok_r(NULL, ",", &saveptr);
+			if (tok) {
+				unsigned long size = 0;
+
+				ret = get_stack_size(tok, &size);
+				param->dump_size = size;
+			}
+#endif /* HAVE_DWARF_UNWIND_SUPPORT */
+		} else if (!strncmp(name, "lbr", sizeof("lbr"))) {
+			if (!strtok_r(NULL, ",", &saveptr)) {
+				param->record_mode = CALLCHAIN_LBR;
+				ret = 0;
+			} else
+				pr_err("callchain: No more arguments "
+					"needed for --call-graph lbr\n");
+			break;
+		} else {
+			pr_err("callchain: Unknown --call-graph option "
+			       "value: %s\n", arg);
+			break;
+		}
+
+	} while (0);
+
+	free(buf);
+	return ret;
+}
+
 int filename__read_str(const char *filename, char **buf, size_t *sizep)
 {
 	size_t size = 0, alloc_size = 0;
diff --git a/tools/perf/util/util.h b/tools/perf/util/util.h
index 20d625a..8148703 100644
--- a/tools/perf/util/util.h
+++ b/tools/perf/util/util.h
@@ -351,4 +351,6 @@ static inline char *asprintf_expr_not_in_ints(const char *var, size_t nints, int
 	return asprintf_expr_inout_ints(var, false, nints, ints);
 }
 
+int get_stack_size(const char *str, unsigned long *_size);
+
 #endif /* GIT_COMPAT_UTIL_H */

^ permalink raw reply related	[flat|nested] 20+ messages in thread

* [tip:perf/core] perf tools: Unset perf_event_attr:: freq when period term is set
  2015-08-08 17:12     ` [PATCH] perf tools: Unset perf_event_attr::freq when period term is set Jiri Olsa
@ 2015-08-12 12:29       ` tip-bot for Jiri Olsa
  0 siblings, 0 replies; 20+ messages in thread
From: tip-bot for Jiri Olsa @ 2015-08-12 12:29 UTC (permalink / raw)
  To: linux-tip-commits
  Cc: jolsa, acme, mingo, linux-kernel, tglx, jolsa, ak, kan.liang,
	namhyung, hpa

Commit-ID:  ab35a7d0ee59a36c9c567defe43c1adb72e9240c
Gitweb:     http://git.kernel.org/tip/ab35a7d0ee59a36c9c567defe43c1adb72e9240c
Author:     Jiri Olsa <jolsa@redhat.com>
AuthorDate: Sat, 8 Aug 2015 19:12:10 +0200
Committer:  Arnaldo Carvalho de Melo <acme@redhat.com>
CommitDate: Mon, 10 Aug 2015 11:58:05 -0300

perf tools: Unset perf_event_attr::freq when period term is set

We need to unset 'perf_event_attr::freq' bit (default 1) when
'period' term is specified within event definition like:

  -e 'cpu/cpu-cycles,call-graph=fp,time,period=100000'

otherwise it will handle the period value as frequency
(and fail if it crossed the maximum allowed frequency value).

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Cc: Andi Kleen <ak@linux.intel.com>
Cc: Kan Liang <kan.liang@intel.com>
Cc: Namhyung Kim <namhyung@kernel.org>
Link: http://lkml.kernel.org/r/20150808171210.GC17040@krava.brq.redhat.com
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
---
 tools/perf/util/evsel.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c
index f572f46..a59710f 100644
--- a/tools/perf/util/evsel.c
+++ b/tools/perf/util/evsel.c
@@ -598,6 +598,7 @@ static void apply_config_terms(struct perf_evsel *evsel)
 		switch (term->type) {
 		case PERF_EVSEL__CONFIG_TERM_PERIOD:
 			attr->sample_period = term->val.period;
+			attr->freq = 0;
 			break;
 		case PERF_EVSEL__CONFIG_TERM_TIME:
 			if (term->val.time)

^ permalink raw reply related	[flat|nested] 20+ messages in thread

end of thread, other threads:[~2015-08-12 12:30 UTC | newest]

Thread overview: 20+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2015-08-06 19:44 [PATCH RFC V9 1/3] perf,tools: move callchain option parse code to util.c kan.liang
2015-08-06 19:44 ` [PATCH RFC V9 2/3] perf,tools: per-event callgraph support kan.liang
2015-08-07 10:54   ` Jiri Olsa
2015-08-07 15:38   ` Arnaldo Carvalho de Melo
2015-08-07 15:49     ` Arnaldo Carvalho de Melo
2015-08-08 16:45       ` Jiri Olsa
2015-08-08 17:35         ` Arnaldo Carvalho de Melo
2015-08-10 13:46         ` Arnaldo Carvalho de Melo
2015-08-10 12:56       ` Liang, Kan
2015-08-10 15:39         ` Arnaldo Carvalho de Melo
2015-08-10 18:57           ` Liang, Kan
2015-08-10 19:39             ` Arnaldo Carvalho de Melo
2015-08-08 17:12     ` [PATCH] perf tools: Unset perf_event_attr::freq when period term is set Jiri Olsa
2015-08-12 12:29       ` [tip:perf/core] perf tools: Unset perf_event_attr:: freq " tip-bot for Jiri Olsa
2015-08-06 19:44 ` [PATCH RFC V9 3/3] perf,tests: Add tests to callgraph and time parse kan.liang
2015-08-07 10:54   ` Jiri Olsa
2015-08-07 10:51 ` [PATCH] perf tools: Move perf_counts struct and functions into separate object Jiri Olsa
2015-08-07 13:14   ` Arnaldo Carvalho de Melo
2015-08-12 12:28   ` [tip:perf/core] perf stat: " tip-bot for Jiri Olsa
2015-08-12 12:29 ` [tip:perf/core] perf callchain: Move option parsing code to util.c tip-bot for Kan Liang

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.