[RFC 0/3] perf tools: Adding xz decompression support

[RFC 0/3] perf tools: Adding xz decompression support

[Jiri Olsa]

hi,
adding support for decompression *.xz objects.

However I'm not sure the way we have now is optimal. For example
the annotation does not work on compressed modules at the moment.

Currently we detect 'supported' suffixes for kernel modules and
update the dso::symtab_type. This info then triggers decompression
of the object when we need the its file descriptor in symsrc__init.

I'm thinking of some generic encapsulation of the decompression,
because it's clearly needed on more places (annotation). I'll try
to come up with som patches, but wanted to hear some ideas first.

Current work available at:
  git://git.kernel.org/pub/scm/linux/kernel/git/jolsa/perf.git
  perf/lzma

thanks,
jirka


Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Corey Ashford <cjashfor@linux.vnet.ibm.com>
Cc: David Ahern <dsahern@gmail.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Josh Stone <jistone@redhat.com>
Cc: Namhyung Kim <namhyung@kernel.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
Jiri Olsa (3):
      perf tools: Update symtab_type with _KMODULE_COMP
      perf tools: Add feature check for lzma library
      perf tools: Add lzma decompression support for kernel module

 tools/perf/Makefile.perf                     |  8 +++++++-
 tools/perf/config/Makefile                   | 15 ++++++++++++--
 tools/perf/config/feature-checks/Makefile    |  6 +++++-
 tools/perf/config/feature-checks/test-lzma.c |  9 +++++++++
 tools/perf/util/dso.c                        |  3 +++
 tools/perf/util/lzma.c                       | 95 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 tools/perf/util/machine.c                    |  9 ++++++++-
 tools/perf/util/util.h                       |  4 ++++
 8 files changed, 144 insertions(+), 5 deletions(-)
 create mode 100644 tools/perf/config/feature-checks/test-lzma.c
 create mode 100644 tools/perf/util/lzma.c

[PATCH 1/3] perf tools: Update symtab_type with _KMODULE_COMP

[Jiri Olsa]

We are missing update of dso->symtab_type when we actualize
modules paths in map_groups__set_modules_path_dir.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Cc: Corey Ashford <cjashfor@linux.vnet.ibm.com>
Cc: David Ahern <dsahern@gmail.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Namhyung Kim <namhyung@kernel.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
---
 tools/perf/util/machine.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c
index 1bca3a9f2b16..e750f829e84e 100644
--- a/tools/perf/util/machine.c
+++ b/tools/perf/util/machine.c
@@ -869,14 +869,17 @@ static int map_groups__set_modules_path_dir(struct map_groups *mg,
 			     dso_name[PATH_MAX];
 			struct map *map;
 			char *long_name;
+			bool compressed = false;
 
 			if (dot == NULL)
 				continue;
 
 			/* On some system, modules are compressed like .ko.gz */
 			if (is_supported_compression(dot + 1) &&
-			    is_kmodule_extension(dot - 2))
+			    is_kmodule_extension(dot - 2)) {
 				dot -= 3;
+				compressed = true;
+			}
 
 			snprintf(dso_name, sizeof(dso_name), "[%.*s]",
 				 (int)(dot - dent->d_name), dent->d_name);
@@ -894,6 +897,10 @@ static int map_groups__set_modules_path_dir(struct map_groups *mg,
 			}
 			dso__set_long_name(map->dso, long_name, true);
 			dso__kernel_module_get_build_id(map->dso, "");
+
+			/* _KMODULE_COMP should be next to _KMODULE */
+			if (compressed)
+				map->dso->symtab_type++;
 		}
 	}
 
-- 
1.9.3

[PATCH 2/3] perf tools: Add feature check for lzma library

[Jiri Olsa]

Will be used to decompress 'xz' objects.

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Cc: Corey Ashford <cjashfor@linux.vnet.ibm.com>
Cc: David Ahern <dsahern@gmail.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Namhyung Kim <namhyung@kernel.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
---
 tools/perf/config/Makefile                   | 6 ++++--
 tools/perf/config/feature-checks/Makefile    | 6 +++++-
 tools/perf/config/feature-checks/test-lzma.c | 9 +++++++++
 3 files changed, 18 insertions(+), 3 deletions(-)
 create mode 100644 tools/perf/config/feature-checks/test-lzma.c

diff --git a/tools/perf/config/Makefile b/tools/perf/config/Makefile
index cc224080b525..d49e1ebb3e41 100644
--- a/tools/perf/config/Makefile
+++ b/tools/perf/config/Makefile
@@ -202,7 +202,8 @@ CORE_FEATURE_TESTS =			\
 	stackprotector-all		\
 	timerfd				\
 	libdw-dwarf-unwind		\
-	zlib
+	zlib				\
+	lzma
 
 LIB_FEATURE_TESTS =			\
 	dwarf				\
@@ -217,7 +218,8 @@ LIB_FEATURE_TESTS =			\
 	libslang			\
 	libunwind			\
 	libdw-dwarf-unwind		\
-	zlib
+	zlib				\
+	lzma
 
 VF_FEATURE_TESTS =			\
 	backtrace			\
diff --git a/tools/perf/config/feature-checks/Makefile b/tools/perf/config/feature-checks/Makefile
index 42ac05aaf8ac..cd0a4eab78ab 100644
--- a/tools/perf/config/feature-checks/Makefile
+++ b/tools/perf/config/feature-checks/Makefile
@@ -31,7 +31,8 @@ FILES=					\
 	test-libdw-dwarf-unwind.bin	\
 	test-compile-32.bin		\
 	test-compile-x32.bin		\
-	test-zlib.bin
+	test-zlib.bin			\
+	test-lzma.bin
 
 CC := $(CROSS_COMPILE)gcc -MD
 PKG_CONFIG := $(CROSS_COMPILE)pkg-config
@@ -147,6 +148,9 @@ test-compile-x32.bin:
 test-zlib.bin:
 	$(BUILD) -lz
 
+test-lzma.bin:
+	$(BUILD) -llzma
+
 -include *.d
 
 ###############################
diff --git a/tools/perf/config/feature-checks/test-lzma.c b/tools/perf/config/feature-checks/test-lzma.c
new file mode 100644
index 000000000000..586bdd0ed87a
--- /dev/null
+++ b/tools/perf/config/feature-checks/test-lzma.c
@@ -0,0 +1,9 @@
+#include <lzma.h>
+
+int main(void)
+{
+	lzma_stream strm = LZMA_STREAM_INIT;
+
+	lzma_stream_decoder(&strm, UINT64_MAX, LZMA_CONCATENATED);
+	return 0;
+}
-- 
1.9.3

[PATCH 3/3] perf tools: Add lzma decompression support for kernel module

[Jiri Olsa]

In short, Fedora compresses kernel modules now, and since perf doesn't
recognize this, it can't find any symbols from modules.

Fedora 21 compresses kernel modules, so we need to recognize
it in perf, currently on Fedora 21 we get:

$ perf report --stdio
No kallsyms or vmlinux with build-id 739edc9776ec741029e6f79af6270f8e2bc4b764 was found
[tg3] with build id 739edc9776ec741029e6f79af6270f8e2bc4b764 not found, continuing without symbols
No kallsyms or vmlinux with build-id cfd4a7ccf7dd229d8a2f111b5c01635f5687b056 was found
[sky2] with build id cfd4a7ccf7dd229d8a2f111b5c01635f5687b056 not found, continuing without symbols
No kallsyms or vmlinux with build-id bb534b79b14143d14bba14640dd79a1a04908e51 was found
[be2net] with build id bb534b79b14143d14bba14640dd79a1a04908e51 not found, continuing without symbols
    24.65%  swapper         [kernel.kallsyms]  [k] cpuidle_enter_state
    23.53%  swapper         [kernel.kallsyms]  [k] intel_idle
     1.07%  swapper         [tg3]              [k] tg3_read32
     0.70%  swapper         [kernel.kallsyms]  [k] nr_iowait_cpu
     0.66%  beah-rhts-task  [unknown]          [.] 0x00007f528c6b5fe1
     0.64%  swapper         [kernel.kallsyms]  [k] calc_load_exit_idle
     0.64%  swapper         [sky2]             [k] sky2_poll

With the fix:

$ perf report --stdio
    27.08%  swapper          [kernel.kallsyms]  [k] intel_idle
    21.48%  swapper          [kernel.kallsyms]  [k] cpuidle_enter_state
     1.29%  kworker/u128:0   [kernel.kallsyms]  [k] finish_task_switch
     1.27%  swapper          [tg3]              [k] tg3_read32
     1.13%  swapper          [kernel.kallsyms]  [k] cpu_startup_entry
     0.98%  swapper          [sky2]             [k] sky2_poll
     0.83%  rcu_sched        [kernel.kallsyms]  [k] _raw_spin_lock

Signed-off-by: Jiri Olsa <jolsa@kernel.org>
Cc: Corey Ashford <cjashfor@linux.vnet.ibm.com>
Cc: David Ahern <dsahern@gmail.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Namhyung Kim <namhyung@kernel.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
---
 tools/perf/Makefile.perf   |  8 +++-
 tools/perf/config/Makefile |  9 +++++
 tools/perf/util/dso.c      |  3 ++
 tools/perf/util/lzma.c     | 95 ++++++++++++++++++++++++++++++++++++++++++++++
 tools/perf/util/util.h     |  4 ++
 5 files changed, 118 insertions(+), 1 deletion(-)
 create mode 100644 tools/perf/util/lzma.c

diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf
index aa6a50447c32..10ee29bff61a 100644
--- a/tools/perf/Makefile.perf
+++ b/tools/perf/Makefile.perf
@@ -67,7 +67,9 @@ include config/utilities.mak
 # Define NO_PERF_READ_VDSOX32 if you do not want to build perf-read-vdsox32
 # for reading the x32 mode 32-bit compatibility VDSO in 64-bit mode
 #
-# Define NO_ZLIB if you do not want to support compressed kernel modules
+# Define NO_ZLIB if you do not want to support compressed (gz) kernel modules
+#
+# Define NO_LZMA if you do not want to support compressed (xz) kernel modules
 
 
 ifeq ($(srctree),)
@@ -601,6 +603,10 @@ ifndef NO_ZLIB
   LIB_OBJS += $(OUTPUT)util/zlib.o
 endif
 
+ifndef NO_LZMA
+  LIB_OBJS += $(OUTPUT)util/lzma.o
+endif
+
 ifdef ASCIIDOC8
   export ASCIIDOC8
 endif
diff --git a/tools/perf/config/Makefile b/tools/perf/config/Makefile
index d49e1ebb3e41..4ecd54ca5b96 100644
--- a/tools/perf/config/Makefile
+++ b/tools/perf/config/Makefile
@@ -624,6 +624,15 @@ ifndef NO_ZLIB
   endif
 endif
 
+ifndef NO_LZMA
+  ifeq ($(feature-lzma), 1)
+    CFLAGS += -DHAVE_LZMA_SUPPORT
+    EXTLIBS += -llzma
+  else
+    NO_LZMA := 1
+  endif
+endif
+
 ifndef NO_BACKTRACE
   ifeq ($(feature-backtrace), 1)
     CFLAGS += -DHAVE_BACKTRACE_SUPPORT
diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c
index c2f7d3b90966..1577b4544da9 100644
--- a/tools/perf/util/dso.c
+++ b/tools/perf/util/dso.c
@@ -148,6 +148,9 @@ static const struct {
 #ifdef HAVE_ZLIB_SUPPORT
 	{ "gz", gzip_decompress_to_file },
 #endif
+#ifdef HAVE_LZMA_SUPPORT
+	{ "xz", lzma_decompress_to_file },
+#endif
 	{ NULL, NULL },
 };
 
diff --git a/tools/perf/util/lzma.c b/tools/perf/util/lzma.c
new file mode 100644
index 000000000000..5629aee4dab4
--- /dev/null
+++ b/tools/perf/util/lzma.c
@@ -0,0 +1,95 @@
+#include <lzma.h>
+#include <stdio.h>
+#include <linux/compiler.h>
+#include "util.h"
+#include "debug.h"
+
+#define BUFSIZE 8192
+
+static const char* lzma_strerror(lzma_ret ret)
+{
+	switch ((int) ret) {
+	case LZMA_MEM_ERROR:
+		return "Memory allocation failed";
+	case LZMA_OPTIONS_ERROR:
+		return "Unsupported decompressor flags";
+	case LZMA_FORMAT_ERROR:
+		return "The input is not in the .xz format";
+	case LZMA_DATA_ERROR:
+		return "Compressed file is corrupt";
+	case LZMA_BUF_ERROR:
+		return "Compressed file is truncated or otherwise corrupt";
+	default:
+		return "Unknown error, possibly a bug";
+	}
+}
+
+int lzma_decompress_to_file(const char *input, int output_fd)
+{
+	lzma_action action = LZMA_RUN;
+	lzma_stream strm   = LZMA_STREAM_INIT;
+	lzma_ret ret;
+
+	u8 buf_in[BUFSIZE];
+	u8 buf_out[BUFSIZE];
+	FILE *infile;
+
+	infile = fopen(input, "rb");
+	if (!infile) {
+		pr_err("lzma: fopen failed on %s: '%s'\n",
+		       input, strerror(errno));
+		return -1;
+	}
+
+	ret = lzma_stream_decoder(&strm, UINT64_MAX, LZMA_CONCATENATED);
+	if (ret != LZMA_OK) {
+		pr_err("lzma: lzma_stream_decoder failed %s (%d)\n",
+			lzma_strerror(ret), ret);
+		return -1;
+	}
+
+	strm.next_in   = NULL;
+	strm.avail_in  = 0;
+	strm.next_out  = buf_out;
+	strm.avail_out = sizeof(buf_out);
+
+	while (1) {
+		if (strm.avail_in == 0 && !feof(infile)) {
+			strm.next_in  = buf_in;
+			strm.avail_in = fread(buf_in, 1, sizeof(buf_in), infile);
+
+			if (ferror(infile)) {
+				pr_err("lzma: read error: %s\n", strerror(errno));
+				return -1;
+			}
+
+			if (feof(infile))
+				action = LZMA_FINISH;
+		}
+
+		ret = lzma_code(&strm, action);
+
+		if (strm.avail_out == 0 || ret == LZMA_STREAM_END) {
+			ssize_t write_size = sizeof(buf_out) - strm.avail_out;
+
+			if (writen(output_fd, buf_out, write_size) != write_size) {
+				pr_err("lzma: write error: %s\n", strerror(errno));
+				return -1;
+			}
+
+			strm.next_out  = buf_out;
+			strm.avail_out = sizeof(buf_out);
+		}
+
+		if (ret != LZMA_OK) {
+			if (ret == LZMA_STREAM_END)
+				return 0;
+
+			pr_err("lzma: failed %s\n", lzma_strerror(ret));
+			return -1;
+		}
+	}
+
+	fclose(infile);
+	return 0;
+}
diff --git a/tools/perf/util/util.h b/tools/perf/util/util.h
index 027a5153495c..28e04bc1557d 100644
--- a/tools/perf/util/util.h
+++ b/tools/perf/util/util.h
@@ -327,4 +327,8 @@ bool find_process(const char *name);
 int gzip_decompress_to_file(const char *input, int output_fd);
 #endif
 
+#ifdef HAVE_LZMA_SUPPORT
+int lzma_decompress_to_file(const char *input, int output_fd);
+#endif
+
 #endif /* GIT_COMPAT_UTIL_H */
-- 
1.9.3