Re: Support for Perf CTF traces now in master (was Re: FW: [RFC 0/5] perf tools: Add perf data CTF conversion)

[Jiri Olsa <jolsa@redhat.com>]

On Wed, Nov 26, 2014 at 06:37:21PM +0100, Sebastian Andrzej Siewior wrote:
> * Alexandre Montplaisir | 2014-11-12 17:14:45 [-0500]:

SNIP

> 
> >This was based on the most recent file format I was aware of, we will
> >update it accordingly if required.
> >
> >Testing welcome!
> 
> I pushed the perf changes I mentioned to 
> 
>   git://git.breakpoint.cc/bigeasy/linux.git ctf_convert_7
> 

from perf data convert point of view (and libbabeltrace).. ;-)

I tried to convert big perf.data (2GB) and my laptop got stuck
for few minutes.. the reason is that perf allocated too much memory

I needed to add occasional bt_ctf_stream_flush into the
processing (patch attached)

What I do now is checking if we processed given amount of events
for the stream and once the value is crossed I flush it.

My question is if there's a way to find out the allocated memory
for the stream?  It'd be nicer to setup maximum allocation size
rather than the number of events.

thanks,
jirka



diff --git a/tools/perf/util/data-bt.c b/tools/perf/util/data-bt.c
index c5720e13d8f1..981e8ff2c32a 100644
--- a/tools/perf/util/data-bt.c
+++ b/tools/perf/util/data-bt.c
@@ -39,10 +39,15 @@ struct evsel_priv {
 
 #define MAX_CPUS	4096
 
+struct ctf_stream {
+	struct bt_ctf_stream *stream;
+	u64 count;
+};
+
 struct ctf_writer {
 	/* writer primitives */
 	struct bt_ctf_writer		*writer;
-	struct bt_ctf_stream		*stream[MAX_CPUS];
+	struct ctf_stream		*stream[MAX_CPUS];
 	struct bt_ctf_stream_class	*stream_class;
 	struct bt_ctf_clock		*clock;
 	u32				last_cpu;
@@ -377,6 +382,73 @@ static int add_generic_values(struct ctf_writer *cw,
 	return 0;
 }
 
+#define STREAM_FLUSH_COUNT 1000
+
+static bool is_flush_needed(struct ctf_stream *cstream)
+{
+	return cstream && (cstream->count >= STREAM_FLUSH_COUNT);
+}
+
+static int flush_stream(struct ctf_writer *cw, int cpu)
+{
+	struct ctf_stream *cstream = cw->stream[cpu];
+	int err = 0;
+
+	if (cstream) {
+		err = bt_ctf_stream_flush(cstream->stream);
+		if (err)
+			pr_err("CTF stream %d flush failed\n", cpu);
+
+		cstream->count = 0;
+	}
+
+	return err;
+}
+
+static int create_stream(struct ctf_writer *cw, int cpu)
+{
+	struct ctf_stream *cstream;
+	struct bt_ctf_field *pkt_ctx;
+	struct bt_ctf_field *cpu_field;
+	struct bt_ctf_stream *stream;
+	u32 i = cpu;
+	int ret;
+
+	cstream = zalloc(sizeof(*cstream));
+	if (!cstream) {
+		pr_err("Failed to allocate ctf stream\n");
+		return -1;
+	}
+
+	stream = bt_ctf_writer_create_stream(cw->writer, cw->stream_class);
+	if (!stream)
+		return -1;
+
+	pkt_ctx = bt_ctf_stream_get_packet_context(stream);
+	if (!pkt_ctx) {
+		pr_err("Failed to obtain packet context\n");
+		return -1;
+	}
+
+	cpu_field = bt_ctf_field_structure_get_field(pkt_ctx, "cpu_id");
+	bt_ctf_field_put(pkt_ctx);
+	if (!cpu_field) {
+		pr_err("Failed to obtain cpu field\n");
+		return -1;
+	}
+
+	ret = bt_ctf_field_unsigned_integer_set_value(cpu_field, i);
+	if (ret) {
+		pr_err("Failed to update CPU number\n");
+		return -1;
+	}
+	bt_ctf_field_put(cpu_field);
+
+	cstream->stream = stream;
+	cw->stream[i] = cstream;
+	return 0;
+}
+
 static int process_sample_event(struct perf_tool *tool,
 				union perf_event *_event __maybe_unused,
 				struct perf_sample *sample,
@@ -431,40 +503,14 @@ static int process_sample_event(struct perf_tool *tool,
 		cpu = 0;
 	}
 
-	if (!cw->stream[cpu]) {
-		struct bt_ctf_field *pkt_ctx;
-		struct bt_ctf_field *cpu_field;
-		struct bt_ctf_stream *stream;
-		u32 i = sample->cpu;
-
-		stream = bt_ctf_writer_create_stream(cw->writer, cw->stream_class);
-		if (!stream)
-			return -1;
-
-		pkt_ctx = bt_ctf_stream_get_packet_context(stream);
-		if (!pkt_ctx) {
-			pr_err("Failed to obtain packet context\n");
-			return -1;
-		}
-
-		cpu_field = bt_ctf_field_structure_get_field(pkt_ctx, "cpu_id");
-		bt_ctf_field_put(pkt_ctx);
-		if (!cpu_field) {
-			pr_err("Failed to obtain cpu field\n");
-			return -1;
-		}
+	if (!cw->stream[cpu] && create_stream(cw, cpu))
+		return -1;
 
-		ret = bt_ctf_field_unsigned_integer_set_value(cpu_field, i);
-		if (ret) {
-			pr_err("Failed to update CPU number\n");
-			return -1;
-		}
-		bt_ctf_field_put(cpu_field);
+	if (is_flush_needed(cw->stream[cpu]))
+		flush_stream(cw, cpu);
 
-		cw->stream[i] = stream;
-	}
-
-	bt_ctf_stream_append_event(cw->stream[cpu], event);
+	cw->stream[cpu]->count++;
+	bt_ctf_stream_append_event(cw->stream[cpu]->stream, event);
 	bt_ctf_event_put(event);
 	return 0;
 }
@@ -745,8 +791,12 @@ static void ctf_writer__cleanup(struct ctf_writer *cw)
 	ctf_writer__cleanup_data(cw);
 
 	bt_ctf_clock_put(cw->clock);
-	for (i = 0; i < MAX_CPUS; i++)
-		bt_ctf_stream_put(cw->stream[i]);
+	for (i = 0; i < MAX_CPUS; i++) {
+		if (cw->stream[i]) {
+			bt_ctf_stream_put(cw->stream[i]->stream);
+			free(cw->stream[i]);
+		}
+	}
 	bt_ctf_stream_class_put(cw->stream_class);
 	bt_ctf_writer_put(cw->writer);
 
@@ -862,6 +912,9 @@ int bt_convert__perf2ctf(const char *input, const char *path)
 	if (!session)
 		goto free_writer;
 
+	ordered_events__set_alloc_size(&session->ordered_events,
+				       100*1024*1024);
+
 	/* CTF writer env/clock setup  */
 	if (ctf_writer__setup_env(cw, session))
 		goto free_session;
@@ -872,15 +925,10 @@ int bt_convert__perf2ctf(const char *input, const char *path)
 
 	err = perf_session__process_events(session, &c.tool);
 	if (!err) {
-		int i;
+		int cpu;
 
-		for (i = 0; i < MAX_CPUS; i++) {
-			if (cw->stream[i]) {
-				err = bt_ctf_stream_flush(cw->stream[i]);
-				if (err)
-					pr_err("CTF stream %d flush failed\n", i);
-			}
-		}
+		for (cpu = 0; cpu < MAX_CPUS; cpu++)
+			flush_stream(cw, cpu);
 	}
 
 	fprintf(stderr,