linux-trace-devel.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: "Tzvetomir Stoyanov (VMware)" <tz.stoyanov@gmail.com>
To: rostedt@goodmis.org
Cc: linux-trace-devel@vger.kernel.org
Subject: [PATCH v5 21/40] trace-cmd library: Add compression of the option section of the trace file
Date: Thu, 10 Jun 2021 14:34:07 +0300	[thread overview]
Message-ID: <20210610113426.257931-22-tz.stoyanov@gmail.com> (raw)
In-Reply-To: <20210610113426.257931-1-tz.stoyanov@gmail.com>

Comperss the option section of the trace file. This section is not big
currently and compressing it does not reduce significantly the size of
the file. This could be useful in the future as new options can be
added, storing a potentially huge amount of data.

Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
 lib/trace-cmd/include/trace-cmd-local.h |  1 +
 lib/trace-cmd/trace-input.c             | 46 +++++++++++------
 lib/trace-cmd/trace-output.c            | 69 +++++++++++++++++--------
 tracecmd/trace-dump.c                   | 16 ++++--
 4 files changed, 92 insertions(+), 40 deletions(-)

diff --git a/lib/trace-cmd/include/trace-cmd-local.h b/lib/trace-cmd/include/trace-cmd-local.h
index 59148930..b77c49ff 100644
--- a/lib/trace-cmd/include/trace-cmd-local.h
+++ b/lib/trace-cmd/include/trace-cmd-local.h
@@ -39,6 +39,7 @@ void tracecmd_zlib_free(void);
 int tracecmd_compress_init(void);
 void tracecmd_compress_free(void);
 
+int out_uncompress_block(struct tracecmd_output *handle);
 int out_compression_start(struct tracecmd_output *handle);
 int out_compression_end(struct tracecmd_output *handle);
 void out_compression_reset(struct tracecmd_output *handle);
diff --git a/lib/trace-cmd/trace-input.c b/lib/trace-cmd/trace-input.c
index b76f2b47..8fff003e 100644
--- a/lib/trace-cmd/trace-input.c
+++ b/lib/trace-cmd/trace-input.c
@@ -2761,23 +2761,29 @@ static int handle_options(struct tracecmd_input *handle)
 	/* By default, use usecs, unless told otherwise */
 	handle->flags |= TRACECMD_FL_IN_USECS;
 	handle->options_start = lseek64(handle->fd, 0, SEEK_CUR);
-
+	if (in_uncompress_block(handle))
+		return -1;
 	for (;;) {
-		if (do_read_check(handle, &option, 2))
-			return -1;
+		ret = do_read_check(handle, &option, 2);
+		if (ret)
+			goto out;
 
 		if (option == TRACECMD_OPTION_DONE)
 			break;
 
 		/* next 4 bytes is the size of the option */
-		if (do_read_check(handle, &size, 4))
-			return -1;
+		ret = do_read_check(handle, &size, 4);
+		if (ret)
+			goto out;
 		size = tep_read_number(handle->pevent, &size, 4);
 		buf = malloc(size);
-		if (!buf)
-			return -ENOMEM;
-		if (do_read_check(handle, buf, size))
-			return -1;
+		if (!buf) {
+			ret = -ENOMEM;
+			goto out;
+		}
+		ret = do_read_check(handle, buf, size);
+		if (ret)
+			goto out;
 
 		switch (option) {
 		case TRACECMD_OPTION_DATE:
@@ -2827,14 +2833,16 @@ static int handle_options(struct tracecmd_input *handle)
 							     buf + 8, 4);
 			ret = tsync_cpu_offsets_load(handle, buf + 12, size - 12);
 			if (ret < 0)
-				return ret;
+				goto out;
 			tracecmd_enable_tsync(handle, true);
 			break;
 		case TRACECMD_OPTION_CPUSTAT:
 			buf[size-1] = '\n';
 			cpustats = realloc(cpustats, cpustats_size + size + 1);
-			if (!cpustats)
-				return -ENOMEM;
+			if (!cpustats) {
+				ret = -ENOMEM;
+				goto out;
+			}
 			memcpy(cpustats + cpustats_size, buf, size);
 			cpustats_size += size;
 			cpustats[cpustats_size] = 0;
@@ -2844,14 +2852,17 @@ static int handle_options(struct tracecmd_input *handle)
 			handle->nr_buffers++;
 			handle->buffers = realloc(handle->buffers,
 						  sizeof(*handle->buffers) * handle->nr_buffers);
-			if (!handle->buffers)
-				return -ENOMEM;
+			if (!handle->buffers) {
+				ret = -ENOMEM;
+				goto out;
+			}
 			buffer = &handle->buffers[handle->nr_buffers - 1];
 			buffer->name = strdup(buf + 8);
 			if (!buffer->name) {
 				free(handle->buffers);
 				handle->buffers = NULL;
-				return -ENOMEM;
+				ret = -ENOMEM;
+				goto out;
 			}
 			offset = *(unsigned long long *)buf;
 			buffer->offset = tep_read_number(handle->pevent, &offset, 8);
@@ -2908,8 +2919,11 @@ static int handle_options(struct tracecmd_input *handle)
 	}
 
 	handle->cpustats = cpustats;
+	ret = 0;
 
-	return 0;
+out:
+	in_uncompress_reset(handle);
+	return ret;
 }
 
 static int read_options_type(struct tracecmd_input *handle)
diff --git a/lib/trace-cmd/trace-output.c b/lib/trace-cmd/trace-output.c
index 1b9cf683..81d97d4a 100644
--- a/lib/trace-cmd/trace-output.c
+++ b/lib/trace-cmd/trace-output.c
@@ -150,6 +150,18 @@ __hidden void out_compression_reset(struct tracecmd_output *handle)
 	handle->do_compress = false;
 }
 
+__hidden int out_uncompress_block(struct tracecmd_output *handle)
+{
+	int ret = 0;
+
+	if (handle->file_version < 7 || !handle->compress)
+		return 0;
+	ret = tracecmd_uncompress_block(handle->compress);
+	if (!ret)
+		handle->do_compress = true;
+	return ret;
+}
+
 __hidden int out_compression_start(struct tracecmd_output *handle)
 {
 	if (handle->file_version < 7 || !handle->compress)
@@ -1326,32 +1338,38 @@ int tracecmd_write_options(struct tracecmd_output *handle)
 
 	if (do_write_check(handle, "options  ", 10))
 		return -1;
-
+	handle->options_start = lseek64(handle->fd, 0, SEEK_CUR);
+	out_compression_start(handle);
 	list_for_each_entry(options, &handle->options, list) {
 		endian2 = convert_endian_2(handle, options->id);
 		if (do_write_check(handle, &endian2, 2))
-			return -1;
+			goto error;
 
 		endian4 = convert_endian_4(handle, options->size);
 		if (do_write_check(handle, &endian4, 4))
-			return -1;
+			goto error;
 
 		/* Save the data location in case it needs to be updated */
-		options->offset = lseek64(handle->fd, 0, SEEK_CUR);
+		options->offset = do_lseek(handle, 0, SEEK_CUR);
 
 		if (do_write_check(handle, options->data,
 				   options->size))
-			return -1;
+			goto error;
 	}
 
 	option = TRACECMD_OPTION_DONE;
 
 	if (do_write_check(handle, &option, 2))
-		return -1;
+		goto error;
+	if (out_compression_end(handle))
+		goto error;
 
 	handle->file_state = TRACECMD_FILE_OPTIONS;
 
 	return 0;
+error:
+	out_compression_reset(handle);
+	return -1;
 }
 
 int tracecmd_append_options(struct tracecmd_output *handle)
@@ -1367,42 +1385,51 @@ int tracecmd_append_options(struct tracecmd_output *handle)
 	 * We can append only if options are already written and tracing data
 	 * is not yet written
 	 */
-	if (handle->file_state != TRACECMD_FILE_OPTIONS)
+	if (handle->file_state != TRACECMD_FILE_OPTIONS || !handle->options_start)
 		return -1;
-
-	if (lseek64(handle->fd, 0, SEEK_END) == (off_t)-1)
+	if (lseek64(handle->fd, handle->options_start, SEEK_SET) == (off_t)-1)
 		return -1;
-	offset = lseek64(handle->fd, -2, SEEK_CUR);
-	if (offset == (off_t)-1)
+	if (out_uncompress_block(handle))
 		return -1;
-
-	r = pread(handle->fd, &option, 2, offset);
+	if (do_lseek(handle, 0, SEEK_END) == -1)
+		goto error;
+	offset = do_lseek(handle, -2, SEEK_CUR);
+	if (offset == (off_t)-1)
+		goto error;
+	r = do_preed(handle, &option, 2, offset);
 	if (r != 2 || option != TRACECMD_OPTION_DONE)
-		return -1;
-
+		goto error;
 	list_for_each_entry(options, &handle->options, list) {
 		endian2 = convert_endian_2(handle, options->id);
 		if (do_write_check(handle, &endian2, 2))
-			return -1;
+			goto error;
 
 		endian4 = convert_endian_4(handle, options->size);
 		if (do_write_check(handle, &endian4, 4))
-			return -1;
+			goto error;
 
 		/* Save the data location in case it needs to be updated */
-		options->offset = lseek64(handle->fd, 0, SEEK_CUR);
+		options->offset = do_lseek(handle, 0, SEEK_CUR);
 
 		if (do_write_check(handle, options->data,
 				   options->size))
-			return -1;
+			goto error;
 	}
-
 	option = TRACECMD_OPTION_DONE;
 
 	if (do_write_check(handle, &option, 2))
-		return -1;
+		goto error;
 
+	if (handle->file_version >= 7 && handle->compress) {
+		if (lseek64(handle->fd, handle->options_start, SEEK_SET) == (off_t)-1)
+			goto error;
+		if (out_compression_end(handle))
+			goto error;
+	}
 	return 0;
+error:
+	out_compression_reset(handle);
+	return -1;
 }
 
 static struct tracecmd_option *
diff --git a/tracecmd/trace-dump.c b/tracecmd/trace-dump.c
index 56acb01d..8d0f2251 100644
--- a/tracecmd/trace-dump.c
+++ b/tracecmd/trace-dump.c
@@ -82,6 +82,13 @@ static int read_compressed(int fd, char *dst, int len)
 	return read_fd(fd, dst, len);
 }
 
+static int do_lseek(int fd, int offset, int whence)
+{
+	if (read_compress)
+		return tracecmd_compress_lseek(compress, offset, whence);
+	return lseek64(fd, offset, whence);
+}
+
 static int read_file_string(int fd, char *dst, int len)
 {
 	size_t size = 0;
@@ -625,6 +632,9 @@ static void dump_options(int fd)
 	unsigned int size;
 	int count = 0;
 
+	if (uncompress_block())
+		die("cannot uncompress file block");
+
 	for (;;) {
 		if (read_file_number(fd, &option, 2))
 			die("cannot read the option id");
@@ -635,7 +645,7 @@ static void dump_options(int fd)
 
 		count++;
 		if (!DUMP_CHECK(OPTIONS) && !DUMP_CHECK(CLOCK) && !DUMP_CHECK(SUMMARY)) {
-			lseek64(fd, size, SEEK_CUR);
+			do_lseek(fd, size, SEEK_CUR);
 			continue;
 		}
 		switch (option) {
@@ -685,12 +695,12 @@ static void dump_options(int fd)
 		default:
 			do_print(OPTIONS, " %d %d\t[Unknown option, size - skipping]\n",
 				 option, size);
-			lseek64(fd, size, SEEK_CUR);
+			do_lseek(fd, size, SEEK_CUR);
 			break;
 		}
 	}
 	do_print(SUMMARY, "\t[%d options]\n", count);
-
+	uncompress_reset();
 }
 
 static void dump_latency(int fd)
-- 
2.31.1


  parent reply	other threads:[~2021-06-10 11:35 UTC|newest]

Thread overview: 44+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-06-10 11:33 [PATCH v5 00/40] Add trace file compression Tzvetomir Stoyanov (VMware)
2021-06-10 11:33 ` [PATCH v5 01/40] trace-cmd library: Remove unused private APIs for creating trace files Tzvetomir Stoyanov (VMware)
2021-06-10 11:33 ` [PATCH v5 02/40] trace-cmd library: Remove unused API tracecmd_update_option Tzvetomir Stoyanov (VMware)
2021-06-10 11:33 ` [PATCH v5 03/40] trace-cmd: Check if file version is supported Tzvetomir Stoyanov (VMware)
2021-06-10 11:33 ` [PATCH v5 04/40] trace-cmd library: Add new API to get file version of input handler Tzvetomir Stoyanov (VMware)
2021-06-10 11:33 ` [PATCH v5 05/40] trace-cmd library: Select the file version when writing trace file Tzvetomir Stoyanov (VMware)
2021-06-10 11:33 ` [PATCH v5 06/40] trace-cmd: Add APIs for library initialization and free Tzvetomir Stoyanov (VMware)
2021-06-22  1:27   ` Steven Rostedt
2021-06-22  6:41     ` Tzvetomir Stoyanov
2021-06-22 13:21       ` Steven Rostedt
2021-06-10 11:33 ` [PATCH v5 07/40] trace-cmd library: Add support for compression algorithms Tzvetomir Stoyanov (VMware)
2021-06-10 11:33 ` [PATCH v5 08/40] trace-cmd list: Show supported " Tzvetomir Stoyanov (VMware)
2021-06-10 11:33 ` [PATCH v5 09/40] trace-cmd library: Bump the trace file version to 7 Tzvetomir Stoyanov (VMware)
2021-06-10 11:33 ` [PATCH v5 10/40] trace-cmd library: Compress part of the trace file Tzvetomir Stoyanov (VMware)
2021-06-10 11:33 ` [PATCH v5 11/40] trace-cmd library: Read compressed " Tzvetomir Stoyanov (VMware)
2021-06-10 11:33 ` [PATCH v5 12/40] trace-cmd library: Add new API to get compression of input handler Tzvetomir Stoyanov (VMware)
2021-06-10 11:33 ` [PATCH v5 13/40] trace-cmd library: Inherit compression algorithm from input file Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 14/40] trace-cmd library: Extend the create file APIs to support different compression Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 15/40] trace-cmd record: Add new parameter --compression Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 16/40] trace-cmd dump: Add support for trace files version 7 Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 17/40] trace-cmd library: Add support for zlib compression library Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 18/40] trace-cmd library: Hide the logic for updating buffer offset Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 19/40] trace-cmd: Move buffers description outside of options Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 20/40] trace-cmd library: Track the offset in the option section in the trace file Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` Tzvetomir Stoyanov (VMware) [this message]
2021-06-10 11:34 ` [PATCH v5 22/40] trace-cmd library: Refactor the logic for writing trace data in the file Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 23/40] trace-cmd library: Add APIs for read and write compressed data in chunks Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 24/40] trace-cmd: Compress trace data Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 25/40] trace-cmd: Read compressed " Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 26/40] trace-cmd library: Compress latency " Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 27/40] trace-cmd: Read compressed " Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 28/40] trace-cmd library: Reuse within the library the function that checks file state Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 29/40] trace-cmd library: Make tracecmd_copy_headers() to work with output handler Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 30/40] trace-cmd: Do not use trace file compression with streams Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 31/40] trace-cmd library: Add new API to get file version of output handler Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 32/40] trace-cmd: Add file state parameter to tracecmd_copy Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 33/40] trace-cmd: Copy CPU count in tracecmd_copy Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 34/40] trace-cmd: Copy buffers description " Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 35/40] trace-cmd: Copy options " Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 36/40] trace-cmd library: Refactor the logic for writing CPU trace data Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 37/40] trace-cmd library: Refactor the logic for writing CPU instance " Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 38/40] trace-cmd: Copy trace data in tracecmd_copy Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 39/40] trace-cmd: Add compression parameter to tracecmd_copy Tzvetomir Stoyanov (VMware)
2021-06-10 11:34 ` [PATCH v5 40/40] trace-cmd: Add new command "trace-cmd convert" Tzvetomir Stoyanov (VMware)

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210610113426.257931-22-tz.stoyanov@gmail.com \
    --to=tz.stoyanov@gmail.com \
    --cc=linux-trace-devel@vger.kernel.org \
    --cc=rostedt@goodmis.org \
    --subject='Re: [PATCH v5 21/40] trace-cmd library: Add compression of the option section of the trace file' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).