linux-trace-devel.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: "Tzvetomir Stoyanov (VMware)" <tz.stoyanov@gmail.com>
To: rostedt@goodmis.org
Cc: linux-trace-devel@vger.kernel.org
Subject: [PATCH v6 10/45] trace-cmd library: Compress part of the trace file
Date: Mon, 14 Jun 2021 10:49:54 +0300	[thread overview]
Message-ID: <20210614075029.598048-11-tz.stoyanov@gmail.com> (raw)
In-Reply-To: <20210614075029.598048-1-tz.stoyanov@gmail.com>

Compress part of the trace.dat file metadata. If there is compression
support, chose file version 7 and compress these parts of the file:
 - ftrace events format
 - format of recorded events
 - information of the mapping of function addresses to the function names
 - trace_printk() format strings
 - information of the mapping a PID to a process name

A new compression header is added in the file, right after the page size
information.

Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
 lib/trace-cmd/include/trace-cmd-local.h |   4 +
 lib/trace-cmd/trace-output.c            | 145 ++++++++++++++++++++----
 2 files changed, 127 insertions(+), 22 deletions(-)

diff --git a/lib/trace-cmd/include/trace-cmd-local.h b/lib/trace-cmd/include/trace-cmd-local.h
index 4d5d1455..93613cd7 100644
--- a/lib/trace-cmd/include/trace-cmd-local.h
+++ b/lib/trace-cmd/include/trace-cmd-local.h
@@ -34,4 +34,8 @@ void tracecmd_info(const char *fmt, ...);
 void tracecmd_compress_init(void);
 void tracecmd_compress_free(void);
 
+int out_compression_start(struct tracecmd_output *handle);
+int out_compression_end(struct tracecmd_output *handle);
+void out_compression_reset(struct tracecmd_output *handle);
+
 #endif /* _TRACE_CMD_LOCAL_H */
diff --git a/lib/trace-cmd/trace-output.c b/lib/trace-cmd/trace-output.c
index bacda23b..87816b37 100644
--- a/lib/trace-cmd/trace-output.c
+++ b/lib/trace-cmd/trace-output.c
@@ -58,6 +58,10 @@ struct tracecmd_output {
 	bool			quiet;
 	unsigned long		file_state;
 	unsigned long		file_version;
+
+	bool			do_compress;
+	struct tracecmd_compression *compress;
+
 	struct list_head	options;
 	struct tracecmd_msg_handle *msg_handle;
 	char			*trace_clock;
@@ -78,12 +82,31 @@ struct list_event_system {
 static stsize_t
 do_write_check(struct tracecmd_output *handle, const void *data, tsize_t size)
 {
+	if (handle->do_compress)
+		return tracecmd_compress_write(handle->compress, data, size);
+
 	if (handle->msg_handle)
 		return tracecmd_msg_data_send(handle->msg_handle, data, size);
 
 	return __do_write_check(handle->fd, data, size);
 }
 
+static inline int do_lseek(struct tracecmd_output *handle, off_t offset, int whence)
+{
+	if (handle->do_compress)
+		return tracecmd_compress_lseek(handle->compress, offset, whence);
+	else
+		return lseek64(handle->fd, offset, whence);
+}
+
+static inline int do_preed(struct tracecmd_output *handle, void *dst, int len, off_t offset)
+{
+	if (handle->do_compress)
+		return tracecmd_compress_pread(handle->compress, dst, len, offset);
+	else
+		return pread(handle->fd, dst, len, offset);
+}
+
 static short convert_endian_2(struct tracecmd_output *handle, short val)
 {
 	if (!handle->pevent)
@@ -109,6 +132,31 @@ static unsigned long long convert_endian_8(struct tracecmd_output *handle,
 	return tep_read_number(handle->pevent, &val, 8);
 }
 
+__hidden void out_compression_reset(struct tracecmd_output *handle)
+{
+	if (handle->file_version < 7)
+		return;
+	tracecmd_compress_reset(handle->compress);
+	handle->do_compress = false;
+}
+
+__hidden int out_compression_start(struct tracecmd_output *handle)
+{
+	if (handle->file_version < 7 || !handle->compress)
+		return 0;
+	tracecmd_compress_reset(handle->compress);
+	handle->do_compress = true;
+	return 0;
+}
+
+__hidden int out_compression_end(struct tracecmd_output *handle)
+{
+	if (handle->file_version < 7)
+		return 0;
+	handle->do_compress = false;
+	return tracecmd_compress_block(handle->compress);
+}
+
 /**
  * tracecmd_set_quiet - Set if to print output to the screen
  * @quiet: If non zero, print no output to the screen
@@ -159,6 +207,7 @@ void tracecmd_output_free(struct tracecmd_output *handle)
 		free(option);
 	}
 	free(handle->trace_clock);
+	tracecmd_compress_destroy(handle->compress);
 	free(handle);
 }
 
@@ -659,13 +708,17 @@ static int read_ftrace_files(struct tracecmd_output *handle)
 	}
 
 	create_event_list_item(handle, &systems, &list);
-
+	out_compression_start(handle);
 	ret = copy_event_system(handle, systems);
+	if (!ret)
+		ret = out_compression_end(handle);
+	else
+		out_compression_reset(handle);
 
 	free_list_events(systems);
 
-	handle->file_state = TRACECMD_FILE_FTRACE_EVENTS;
-
+	if (!ret)
+		handle->file_state = TRACECMD_FILE_FTRACE_EVENTS;
 	return ret;
 }
 
@@ -715,6 +768,7 @@ static int read_event_files(struct tracecmd_output *handle,
 	for (slist = systems; slist; slist = slist->next)
 		count++;
 
+	out_compression_start(handle);
 	ret = -1;
 	endian4 = convert_endian_4(handle, count);
 	if (do_write_check(handle, &endian4, 4))
@@ -730,8 +784,14 @@ static int read_event_files(struct tracecmd_output *handle,
 		ret = copy_event_system(handle, slist);
 	}
 
-	handle->file_state = TRACECMD_FILE_ALL_EVENTS;
+	if (!ret)
+		ret = out_compression_end(handle);
  out_free:
+	if (!ret)
+		handle->file_state = TRACECMD_FILE_ALL_EVENTS;
+	else
+		out_compression_reset(handle);
+
 	free_list_events(systems);
 
 	return ret;
@@ -793,20 +853,20 @@ static int read_proc_kallsyms(struct tracecmd_output *handle,
 
 	if (kallsyms)
 		path = kallsyms;
-
+	out_compression_start(handle);
 	ret = stat(path, &st);
 	if (ret < 0) {
 		/* not found */
 		size = 0;
 		endian4 = convert_endian_4(handle, size);
-		if (do_write_check(handle, &endian4, 4))
-			return -1;
-		return 0;
+		ret = do_write_check(handle, &endian4, 4);
+		goto out;
 	}
 	size = get_size(path);
 	endian4 = convert_endian_4(handle, size);
-	if (do_write_check(handle, &endian4, 4))
-		return -1;
+	ret = do_write_check(handle, &endian4, 4);
+	if (ret)
+		goto out;
 
 	set_proc_kptr_restrict(0);
 	check_size = copy_file(handle, path);
@@ -814,13 +874,18 @@ static int read_proc_kallsyms(struct tracecmd_output *handle,
 		errno = EINVAL;
 		tracecmd_warning("error in size of file '%s'", path);
 		set_proc_kptr_restrict(1);
-		return -1;
+		ret = -1;
+		goto out;
 	}
 	set_proc_kptr_restrict(1);
 
-	handle->file_state = TRACECMD_FILE_KALLSYMS;
-
-	return 0;
+	ret = out_compression_end(handle);
+out:
+	if (!ret)
+		handle->file_state = TRACECMD_FILE_KALLSYMS;
+	else
+		out_compression_reset(handle);
+	return ret;
 }
 
 static int read_ftrace_printk(struct tracecmd_output *handle)
@@ -840,6 +905,7 @@ static int read_ftrace_printk(struct tracecmd_output *handle)
 	if (!path)
 		return -1;
 
+	out_compression_start(handle);
 	ret = stat(path, &st);
 	if (ret < 0) {
 		/* not found */
@@ -861,11 +927,14 @@ static int read_ftrace_printk(struct tracecmd_output *handle)
 	}
 
  out:
-	handle->file_state = TRACECMD_FILE_PRINTK;
 	put_tracing_file(path);
+	if (out_compression_end(handle))
+		return -1;
+	handle->file_state = TRACECMD_FILE_PRINTK;
 	return 0;
  fail:
 	put_tracing_file(path);
+	out_compression_reset(handle);
 	return -1;
 }
 
@@ -911,14 +980,39 @@ out_free:
 static int select_file_version(struct tracecmd_output *handle,
 				struct tracecmd_input *ihandle)
 {
-	if (ihandle)
+	if (ihandle) {
 		handle->file_version = tracecmd_get_in_file_version(ihandle);
-	else
-		handle->file_version = FILE_VERSION;
+	} else {
+		handle->compress = tracecmd_compress_alloc(NULL, NULL, handle->fd,
+							   handle->pevent, handle->msg_handle);
+		if (handle->compress)
+			handle->file_version = 7;
+		else
+			handle->file_version = 6;
+	}
 
 	return 0;
 }
 
+static int write_compression_header(struct tracecmd_output *handle)
+{
+	const char *name = NULL;
+	const char *ver = NULL;
+	char *buf;
+	int ret;
+
+	ret = tracecmd_compress_proto_get_name(handle->compress, &name, &ver);
+	if (ret < 0 || !name || !ver)
+		return -1;
+	ret = asprintf(&buf, "%s %s", name, ver);
+	if (ret < 0)
+		return -1;
+	ret = do_write_check(handle, buf, strlen(buf) + 1);
+
+	free(buf);
+	return ret;
+}
+
 static struct tracecmd_output *
 create_file_fd(int fd, struct tracecmd_input *ihandle,
 	       const char *tracing_dir,
@@ -992,8 +1086,9 @@ create_file_fd(int fd, struct tracecmd_input *ihandle,
 	endian4 = convert_endian_4(handle, handle->page_size);
 	if (do_write_check(handle, &endian4, 4))
 		goto out_free;
+	if (handle->file_version >= 7 && write_compression_header(handle))
+		goto out_free;
 	handle->file_state = TRACECMD_FILE_INIT;
-
 	if (ihandle)
 		return handle;
 
@@ -1285,11 +1380,17 @@ int tracecmd_write_cmdlines(struct tracecmd_output *handle)
 				 handle->file_state);
 		return ret;
 	}
+	out_compression_start(handle);
+
 	ret = save_tracing_file_data(handle, "saved_cmdlines");
-	if (ret < 0)
+	if (ret < 0) {
+		out_compression_reset(handle);
 		return ret;
-	handle->file_state = TRACECMD_FILE_CMD_LINES;
-	return 0;
+	}
+	ret = out_compression_end(handle);
+	if (!ret)
+		handle->file_state = TRACECMD_FILE_CMD_LINES;
+	return ret;
 }
 
 struct tracecmd_output *tracecmd_create_file_latency(const char *output_file, int cpus)
-- 
2.31.1


  parent reply	other threads:[~2021-06-14  7:50 UTC|newest]

Thread overview: 64+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-06-14  7:49 [PATCH v6 00/45] Add trace file compression Tzvetomir Stoyanov (VMware)
2021-06-14  7:49 ` [PATCH v6 01/45] trace-cmd library: Remove unused private APIs for creating trace files Tzvetomir Stoyanov (VMware)
2021-06-14  7:49 ` [PATCH v6 02/45] trace-cmd library: Remove unused API tracecmd_update_option Tzvetomir Stoyanov (VMware)
2021-06-14  7:49 ` [PATCH v6 03/45] trace-cmd: Check if file version is supported Tzvetomir Stoyanov (VMware)
2021-06-21 22:27   ` Steven Rostedt
2021-06-21 22:36     ` Steven Rostedt
2021-06-14  7:49 ` [PATCH v6 04/45] trace-cmd library: Add new API to get file version of input handler Tzvetomir Stoyanov (VMware)
2021-06-14  7:49 ` [PATCH v6 05/45] trace-cmd library: Select the file version when writing trace file Tzvetomir Stoyanov (VMware)
2021-06-14  7:49 ` [PATCH v6 06/45] trace-cmd: Add APIs for library initialization and free Tzvetomir Stoyanov (VMware)
2021-06-14  7:49 ` [PATCH v6 07/45] trace-cmd library: Add support for compression algorithms Tzvetomir Stoyanov (VMware)
2021-06-14  7:49 ` [PATCH v6 08/45] trace-cmd list: Show supported " Tzvetomir Stoyanov (VMware)
2021-06-14  7:49 ` [PATCH v6 09/45] trace-cmd library: Bump the trace file version to 7 Tzvetomir Stoyanov (VMware)
2021-06-14  7:49 ` Tzvetomir Stoyanov (VMware) [this message]
2021-06-14  7:49 ` [PATCH v6 11/45] trace-cmd library: Read compressed trace file Tzvetomir Stoyanov (VMware)
2021-06-14  7:49 ` [PATCH v6 12/45] trace-cmd library: Add new API to get compression of input handler Tzvetomir Stoyanov (VMware)
2021-06-14  7:49 ` [PATCH v6 13/45] trace-cmd library: Inherit compression algorithm from input file Tzvetomir Stoyanov (VMware)
2021-06-14  7:49 ` [PATCH v6 14/45] trace-cmd library: Extend the create file APIs to support different compression Tzvetomir Stoyanov (VMware)
2021-06-14  7:49 ` [PATCH v6 15/45] trace-cmd record: Add new parameter --compression Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 16/45] trace-cmd dump: Add support for trace files version 7 Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 17/45] trace-cmd library: Add support for zlib compression library Tzvetomir Stoyanov (VMware)
2021-06-22  1:26   ` Steven Rostedt
2021-06-22 10:29     ` Tzvetomir Stoyanov
2021-06-22 13:31       ` Steven Rostedt
2021-06-14  7:50 ` [PATCH v6 18/45] trace-cmd library: Hide the logic for updating buffer offset Tzvetomir Stoyanov (VMware)
2021-06-22  2:10   ` Steven Rostedt
2021-06-14  7:50 ` [PATCH v6 19/45] trace-cmd: Move buffers description outside of options Tzvetomir Stoyanov (VMware)
2021-06-21 23:07   ` Steven Rostedt
2021-06-14  7:50 ` [PATCH v6 20/45] trace-cmd library: Track the offset in the option section in the trace file Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 21/45] trace-cmd library: Add compression of the option section of " Tzvetomir Stoyanov (VMware)
2021-06-21 23:10   ` Steven Rostedt
2021-06-22 10:43     ` Tzvetomir Stoyanov
2021-06-14  7:50 ` [PATCH v6 22/45] trace-cmd library: Refactor the logic for writing trace data in the file Tzvetomir Stoyanov (VMware)
2021-06-21 23:12   ` Steven Rostedt
2021-06-14  7:50 ` [PATCH v6 23/45] trace-cmd library: Add APIs for read and write compressed data in chunks Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 24/45] trace-cmd: Compress trace data Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 25/45] trace-cmd: Read compressed " Tzvetomir Stoyanov (VMware)
2021-06-21 23:23   ` Steven Rostedt
2021-06-22 10:50     ` Tzvetomir Stoyanov
2021-06-22 13:51       ` Steven Rostedt
2021-06-14  7:50 ` [PATCH v6 26/45] trace-cmd library: Compress latency " Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 27/45] trace-cmd: Read compressed " Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 28/45] trace-cmd library: Reuse within the library the function that checks file state Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 29/45] trace-cmd library: Make tracecmd_copy_headers() to work with output handler Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 30/45] trace-cmd: Do not use trace file compression with streams Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 31/45] trace-cmd library: Add new API to get file version of output handler Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 32/45] trace-cmd: Add file state parameter to tracecmd_copy Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 33/45] trace-cmd: Copy CPU count in tracecmd_copy Tzvetomir Stoyanov (VMware)
2021-06-21 23:27   ` Steven Rostedt
2021-06-14  7:50 ` [PATCH v6 34/45] trace-cmd: Copy buffers description " Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 35/45] trace-cmd: Copy options " Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 36/45] trace-cmd library: Refactor the logic for writing CPU trace data Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 37/45] trace-cmd library: Refactor the logic for writing CPU instance " Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 38/45] trace-cmd: Copy trace data in tracecmd_copy Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 39/45] trace-cmd: Add compression parameter to tracecmd_copy Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 40/45] trace-cmd: Add new command "trace-cmd convert" Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 41/45] trace-cmd record: Update man page Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 42/45] trace-cmd: Add convert " Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 43/45] trace-cmd: Update bash completion Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 44/45] trace-cmd list: Update the man page Tzvetomir Stoyanov (VMware)
2021-06-14  7:50 ` [PATCH v6 45/45] trace-cmd: Update trace.dat " Tzvetomir Stoyanov (VMware)
2021-06-22  0:37   ` Steven Rostedt
2021-06-22 11:05     ` Tzvetomir Stoyanov
2021-06-22 13:58       ` Steven Rostedt
2021-06-22  2:22 ` [PATCH v6 00/45] Add trace file compression Steven Rostedt

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210614075029.598048-11-tz.stoyanov@gmail.com \
    --to=tz.stoyanov@gmail.com \
    --cc=linux-trace-devel@vger.kernel.org \
    --cc=rostedt@goodmis.org \
    --subject='Re: [PATCH v6 10/45] trace-cmd library: Compress part of the trace file' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).