* [PATCH v3 0/6] Add VM kernel tracing over vsock sockets
@ 2019-01-14 15:27 Slavomir Kaslev
2019-01-14 15:27 ` [PATCH v3 1/6] trace-cmd: Minor refactoring Slavomir Kaslev
` (5 more replies)
0 siblings, 6 replies; 12+ messages in thread
From: Slavomir Kaslev @ 2019-01-14 15:27 UTC (permalink / raw)
To: linux-trace-devel; +Cc: rostedt, ykaradzhov, tstoyanov
This patchset adds support for tracing guest kernels to trace-cmd.
Changes in v3:
- addressed Steven's feedback
- detect and disable guest tracing if <linux/vm_sockets.h> is not available
- the --date flag is now treated as global for all guest instances
- fixed a bug that caused --date to be ignored for host tracing data when tracing guests
Changes in v2:
- rebased on top of protocol V3
- fixed system clock timestamps with the --date flag
Slavomir Kaslev (5):
trace-cmd: Minor refactoring
trace-cmd: Add tracecmd_create_recorder_virt function
trace-cmd: Add TRACE_REQ and TRACE_RESP messages
trace-cmd: Add buffer instance flags for tracing in guest and agent
context
trace-cmd: Add VM kernel tracing over vsock sockets transport
Steven Rostedt (VMware) (1):
trace-cmd: Detect if vsock sockets are available
Makefile | 7 +
include/trace-cmd/trace-cmd.h | 13 +
lib/trace-cmd/trace-recorder.c | 53 ++-
tracecmd/Makefile | 6 +-
tracecmd/include/trace-local.h | 19 +
tracecmd/trace-agent.c | 229 ++++++++++
tracecmd/trace-cmd.c | 3 +
tracecmd/trace-msg.c | 199 ++++++++-
tracecmd/trace-record.c | 757 ++++++++++++++++++++++++++++-----
tracecmd/trace-usage.c | 13 +-
10 files changed, 1185 insertions(+), 114 deletions(-)
create mode 100644 tracecmd/trace-agent.c
--
2.19.1
^ permalink raw reply [flat|nested] 12+ messages in thread
* [PATCH v3 1/6] trace-cmd: Minor refactoring
2019-01-14 15:27 [PATCH v3 0/6] Add VM kernel tracing over vsock sockets Slavomir Kaslev
@ 2019-01-14 15:27 ` Slavomir Kaslev
2019-01-14 15:27 ` [PATCH v3 2/6] trace-cmd: Detect if vsock sockets are available Slavomir Kaslev
` (4 subsequent siblings)
5 siblings, 0 replies; 12+ messages in thread
From: Slavomir Kaslev @ 2019-01-14 15:27 UTC (permalink / raw)
To: linux-trace-devel; +Cc: rostedt, ykaradzhov, tstoyanov
Pass `struct common_record_context` in trace-record.c instead of explicitly
passing necessary options through additional arguments.
Signed-off-by: Slavomir Kaslev <kaslevs@vmware.com>
---
tracecmd/trace-record.c | 91 +++++++++++++++++++++--------------------
1 file changed, 46 insertions(+), 45 deletions(-)
diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index 01330ee..a8c3464 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -191,6 +191,36 @@ enum {
RESET_HIGH_PRIO = 100000,
};
+enum trace_cmd {
+ CMD_extract,
+ CMD_start,
+ CMD_stream,
+ CMD_profile,
+ CMD_record,
+ CMD_record_agent,
+};
+
+struct common_record_context {
+ enum trace_cmd curr_cmd;
+ struct buffer_instance *instance;
+ const char *output;
+ char *date2ts;
+ char *max_graph_depth;
+ int data_flags;
+
+ int record_all;
+ int total_disable;
+ int disable;
+ int events;
+ int global;
+ int filtered;
+ int date;
+ int manual;
+ int topt;
+ int do_child;
+ int run_command;
+};
+
static void add_reset_file(const char *file, const char *val, int prio)
{
struct reset_file *reset;
@@ -2880,10 +2910,10 @@ again:
return msg_handle;
}
-static void add_options(struct tracecmd_output *handle, char *date2ts, int flags);
+static void add_options(struct tracecmd_output *handle, struct common_record_context *ctx);
static struct tracecmd_msg_handle *
-setup_connection(struct buffer_instance *instance, char *date2ts, int flags)
+setup_connection(struct buffer_instance *instance, struct common_record_context *ctx)
{
struct tracecmd_msg_handle *msg_handle;
struct tracecmd_output *network_handle;
@@ -2893,7 +2923,7 @@ setup_connection(struct buffer_instance *instance, char *date2ts, int flags)
/* Now create the handle through this socket */
if (msg_handle->version == V3_PROTOCOL) {
network_handle = tracecmd_create_init_fd_msg(msg_handle, listed_events);
- add_options(network_handle, date2ts, flags);
+ add_options(network_handle, ctx);
tracecmd_write_cpus(network_handle, instance->cpu_count);
tracecmd_write_options(network_handle);
tracecmd_msg_finish_sending_data(msg_handle);
@@ -2915,7 +2945,7 @@ static void finish_network(struct tracecmd_msg_handle *msg_handle)
free(host);
}
-void start_threads(enum trace_type type, int global, char *date2ts, int flags)
+void start_threads(enum trace_type type, struct common_record_context *ctx)
{
struct buffer_instance *instance;
int *brass = NULL;
@@ -2937,7 +2967,7 @@ void start_threads(enum trace_type type, int global, char *date2ts, int flags)
int x, pid;
if (host) {
- instance->msg_handle = setup_connection(instance, date2ts, flags);
+ instance->msg_handle = setup_connection(instance, ctx);
if (!instance->msg_handle)
die("Failed to make connection");
}
@@ -2952,7 +2982,7 @@ void start_threads(enum trace_type type, int global, char *date2ts, int flags)
brass[0],
instance->cpu_count,
hooks, handle_init,
- global);
+ ctx->global);
if (!pids[i].stream)
die("Creating stream for %d", i);
} else
@@ -3091,19 +3121,19 @@ enum {
DATA_FL_OFFSET = 2,
};
-static void add_options(struct tracecmd_output *handle, char *date2ts, int flags)
+static void add_options(struct tracecmd_output *handle, struct common_record_context *ctx)
{
int type = 0;
- if (date2ts) {
- if (flags & DATA_FL_DATE)
+ if (ctx->date2ts) {
+ if (ctx->data_flags & DATA_FL_DATE)
type = TRACECMD_OPTION_DATE;
- else if (flags & DATA_FL_OFFSET)
+ else if (ctx->data_flags & DATA_FL_OFFSET)
type = TRACECMD_OPTION_OFFSET;
}
if (type)
- tracecmd_add_option(handle, type, strlen(date2ts)+1, date2ts);
+ tracecmd_add_option(handle, type, strlen(ctx->date2ts)+1, ctx->date2ts);
tracecmd_add_option(handle, TRACECMD_OPTION_TRACECLOCK, 0, NULL);
add_option_hooks(handle);
@@ -3111,7 +3141,7 @@ static void add_options(struct tracecmd_output *handle, char *date2ts, int flags
}
-static void record_data(char *date2ts, int flags)
+static void record_data(struct common_record_context *ctx)
{
struct tracecmd_option **buffer_options;
struct tracecmd_output *handle;
@@ -3166,7 +3196,7 @@ static void record_data(char *date2ts, int flags)
if (!handle)
die("Error creating output file");
- add_options(handle, date2ts, flags);
+ add_options(handle, ctx);
/* Only record the top instance under TRACECMD_OPTION_CPUSTAT*/
if (!no_top_instance() && !top_instance.msg_handle) {
@@ -4469,35 +4499,6 @@ void trace_reset(int argc, char **argv)
exit(0);
}
-enum trace_cmd {
- CMD_extract,
- CMD_start,
- CMD_stream,
- CMD_profile,
- CMD_record
-};
-
-struct common_record_context {
- enum trace_cmd curr_cmd;
- struct buffer_instance *instance;
- const char *output;
- char *date2ts;
- char *max_graph_depth;
- int data_flags;
-
- int record_all;
- int total_disable;
- int disable;
- int events;
- int global;
- int filtered;
- int date;
- int manual;
- int topt;
- int do_child;
- int run_command;
-};
-
static void init_common_record_context(struct common_record_context *ctx,
enum trace_cmd curr_cmd)
{
@@ -4986,7 +4987,7 @@ static void record_trace(int argc, char **argv,
if (type & (TRACE_TYPE_RECORD | TRACE_TYPE_STREAM)) {
signal(SIGINT, finish);
if (!latency)
- start_threads(type, ctx->global, ctx->date2ts, ctx->data_flags);
+ start_threads(type, ctx);
} else {
update_task_filter();
tracecmd_enable_tracing();
@@ -5017,7 +5018,7 @@ static void record_trace(int argc, char **argv,
tracecmd_disable_all_tracing(0);
if (IS_RECORD(ctx)) {
- record_data(ctx->date2ts, ctx->data_flags);
+ record_data(ctx);
delete_thread_data();
} else
print_stats();
@@ -5097,7 +5098,7 @@ void trace_extract(int argc, char **argv)
ctx.date2ts = get_date_to_ts();
}
- record_data(ctx.date2ts, ctx.data_flags);
+ record_data(&ctx);
delete_thread_data();
destroy_stats();
finalize_record_trace(&ctx);
--
2.19.1
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH v3 2/6] trace-cmd: Detect if vsock sockets are available
2019-01-14 15:27 [PATCH v3 0/6] Add VM kernel tracing over vsock sockets Slavomir Kaslev
2019-01-14 15:27 ` [PATCH v3 1/6] trace-cmd: Minor refactoring Slavomir Kaslev
@ 2019-01-14 15:27 ` Slavomir Kaslev
2019-01-14 15:27 ` [PATCH v3 3/6] trace-cmd: Add tracecmd_create_recorder_virt function Slavomir Kaslev
` (3 subsequent siblings)
5 siblings, 0 replies; 12+ messages in thread
From: Slavomir Kaslev @ 2019-01-14 15:27 UTC (permalink / raw)
To: linux-trace-devel; +Cc: rostedt, ykaradzhov, tstoyanov
From: "Steven Rostedt (VMware)" <rostedt@goodmis.org>
Detect and define VSOCK if vsock sockets are available on the system.
This macro is used to disable VM remote tracing features on older kernels.
Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
Signed-off-by: Slavomir Kaslev <kaslevs@vmware.com>
---
Makefile | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/Makefile b/Makefile
index 27d3683..abc4f0c 100644
--- a/Makefile
+++ b/Makefile
@@ -205,6 +205,13 @@ CFLAGS ?= -g -Wall
CPPFLAGS ?=
LDFLAGS ?=
+VSOCK_DEFINED := $(shell if (echo "\#include <linux/vm_sockets.h>" | $(CC) -E - >/dev/null 2>&1) ; then echo 1; else echo 0 ; fi)
+
+export VSOCK_DEFINED
+ifeq ($(VSOCK_DEFINED), 1)
+CFLAGS += -DVSOCK
+endif
+
export CFLAGS
export INCLUDES
--
2.19.1
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH v3 3/6] trace-cmd: Add tracecmd_create_recorder_virt function
2019-01-14 15:27 [PATCH v3 0/6] Add VM kernel tracing over vsock sockets Slavomir Kaslev
2019-01-14 15:27 ` [PATCH v3 1/6] trace-cmd: Minor refactoring Slavomir Kaslev
2019-01-14 15:27 ` [PATCH v3 2/6] trace-cmd: Detect if vsock sockets are available Slavomir Kaslev
@ 2019-01-14 15:27 ` Slavomir Kaslev
2019-01-14 22:10 ` Steven Rostedt
2019-01-14 15:27 ` [PATCH v3 4/6] trace-cmd: Add TRACE_REQ and TRACE_RESP messages Slavomir Kaslev
` (2 subsequent siblings)
5 siblings, 1 reply; 12+ messages in thread
From: Slavomir Kaslev @ 2019-01-14 15:27 UTC (permalink / raw)
To: linux-trace-devel; +Cc: rostedt, ykaradzhov, tstoyanov
Add tracecmd_create_recorder_virt function that will be used for
tracing VM guests.
Signed-off-by: Slavomir Kaslev <kaslevs@vmware.com>
---
include/trace-cmd/trace-cmd.h | 1 +
lib/trace-cmd/trace-recorder.c | 53 ++++++++++++++++++++++++----------
2 files changed, 39 insertions(+), 15 deletions(-)
diff --git a/include/trace-cmd/trace-cmd.h b/include/trace-cmd/trace-cmd.h
index 99a455c..f0747c5 100644
--- a/include/trace-cmd/trace-cmd.h
+++ b/include/trace-cmd/trace-cmd.h
@@ -277,6 +277,7 @@ enum {
void tracecmd_free_recorder(struct tracecmd_recorder *recorder);
struct tracecmd_recorder *tracecmd_create_recorder(const char *file, int cpu, unsigned flags);
struct tracecmd_recorder *tracecmd_create_recorder_fd(int fd, int cpu, unsigned flags);
+struct tracecmd_recorder *tracecmd_create_recorder_virt(const char *file, int cpu, int trace_fd);
struct tracecmd_recorder *tracecmd_create_recorder_maxkb(const char *file, int cpu, unsigned flags, int maxkb);
struct tracecmd_recorder *tracecmd_create_buffer_recorder_fd(int fd, int cpu, unsigned flags, const char *buffer);
struct tracecmd_recorder *tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags, const char *buffer);
diff --git a/lib/trace-cmd/trace-recorder.c b/lib/trace-cmd/trace-recorder.c
index 5331925..497f752 100644
--- a/lib/trace-cmd/trace-recorder.c
+++ b/lib/trace-cmd/trace-recorder.c
@@ -148,16 +148,22 @@ tracecmd_create_buffer_recorder_fd2(int fd, int fd2, int cpu, unsigned flags,
recorder->fd1 = fd;
recorder->fd2 = fd2;
- if (flags & TRACECMD_RECORD_SNAPSHOT)
- ret = asprintf(&path, "%s/per_cpu/cpu%d/snapshot_raw", buffer, cpu);
- else
- ret = asprintf(&path, "%s/per_cpu/cpu%d/trace_pipe_raw", buffer, cpu);
- if (ret < 0)
- goto out_free;
+ if (buffer) {
+ if (flags & TRACECMD_RECORD_SNAPSHOT)
+ ret = asprintf(&path, "%s/per_cpu/cpu%d/snapshot_raw",
+ buffer, cpu);
+ else
+ ret = asprintf(&path, "%s/per_cpu/cpu%d/trace_pipe_raw",
+ buffer, cpu);
+ if (ret < 0)
+ goto out_free;
+
+ recorder->trace_fd = open(path, O_RDONLY);
+ free(path);
- recorder->trace_fd = open(path, O_RDONLY);
- if (recorder->trace_fd < 0)
- goto out_free;
+ if (recorder->trace_fd < 0)
+ goto out_free;
+ }
if ((recorder->flags & TRACECMD_RECORD_NOSPLICE) == 0) {
ret = pipe(recorder->brass);
@@ -177,13 +183,9 @@ tracecmd_create_buffer_recorder_fd2(int fd, int fd2, int cpu, unsigned flags,
recorder->pipe_size = pipe_size;
}
- free(path);
-
return recorder;
out_free:
- free(path);
-
tracecmd_free_recorder(recorder);
return NULL;
}
@@ -194,8 +196,9 @@ tracecmd_create_buffer_recorder_fd(int fd, int cpu, unsigned flags, const char *
return tracecmd_create_buffer_recorder_fd2(fd, -1, cpu, flags, buffer, 0);
}
-struct tracecmd_recorder *
-tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags, const char *buffer)
+static struct tracecmd_recorder *
+__tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags,
+ const char *buffer)
{
struct tracecmd_recorder *recorder;
int fd;
@@ -258,6 +261,26 @@ tracecmd_create_buffer_recorder_maxkb(const char *file, int cpu, unsigned flags,
goto out;
}
+struct tracecmd_recorder *
+tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags,
+ const char *buffer)
+{
+ return __tracecmd_create_buffer_recorder(file, cpu, flags, buffer);
+}
+
+struct tracecmd_recorder *
+tracecmd_create_recorder_virt(const char *file, int cpu, int trace_fd)
+{
+ struct tracecmd_recorder *recorder;
+
+ recorder = __tracecmd_create_buffer_recorder(
+ file, cpu, TRACECMD_RECORD_NOSPLICE, NULL);
+ if (recorder)
+ recorder->trace_fd = trace_fd;
+
+ return recorder;
+}
+
struct tracecmd_recorder *tracecmd_create_recorder_fd(int fd, int cpu, unsigned flags)
{
const char *tracing;
--
2.19.1
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH v3 4/6] trace-cmd: Add TRACE_REQ and TRACE_RESP messages
2019-01-14 15:27 [PATCH v3 0/6] Add VM kernel tracing over vsock sockets Slavomir Kaslev
` (2 preceding siblings ...)
2019-01-14 15:27 ` [PATCH v3 3/6] trace-cmd: Add tracecmd_create_recorder_virt function Slavomir Kaslev
@ 2019-01-14 15:27 ` Slavomir Kaslev
2019-01-14 15:27 ` [PATCH v3 5/6] trace-cmd: Add buffer instance flags for tracing in guest and agent context Slavomir Kaslev
2019-01-14 15:28 ` [PATCH v3 6/6] trace-cmd: Add VM kernel tracing over vsock sockets transport Slavomir Kaslev
5 siblings, 0 replies; 12+ messages in thread
From: Slavomir Kaslev @ 2019-01-14 15:27 UTC (permalink / raw)
To: linux-trace-devel; +Cc: rostedt, ykaradzhov, tstoyanov
Add TRACE_REQ and TRACE_RESP messages which are used for initiating guest VM
tracing.
Signed-off-by: Slavomir Kaslev <kaslevs@vmware.com>
---
include/trace-cmd/trace-cmd.h | 12 ++
tracecmd/trace-msg.c | 199 +++++++++++++++++++++++++++++++++-
2 files changed, 210 insertions(+), 1 deletion(-)
diff --git a/include/trace-cmd/trace-cmd.h b/include/trace-cmd/trace-cmd.h
index f0747c5..484340a 100644
--- a/include/trace-cmd/trace-cmd.h
+++ b/include/trace-cmd/trace-cmd.h
@@ -337,6 +337,18 @@ int tracecmd_msg_collect_data(struct tracecmd_msg_handle *msg_handle, int ofd);
bool tracecmd_msg_done(struct tracecmd_msg_handle *msg_handle);
void tracecmd_msg_set_done(struct tracecmd_msg_handle *msg_handle);
+int tracecmd_msg_send_trace_req(struct tracecmd_msg_handle *msg_handle,
+ int argc, char **argv);
+int tracecmd_msg_recv_trace_req(struct tracecmd_msg_handle *msg_handle,
+ int *argc, char ***argv);
+
+int tracecmd_msg_send_trace_resp(struct tracecmd_msg_handle *msg_handle,
+ int nr_cpus, int page_size, int *ports);
+int tracecmd_msg_recv_trace_resp(struct tracecmd_msg_handle *msg_handle,
+ int *nr_cpus, int *page_size, int **ports);
+
+int tracecmd_msg_wait_close(struct tracecmd_msg_handle *msg_handle);
+
/* --- Plugin handling --- */
extern struct tep_plugin_option trace_ftrace_options[];
diff --git a/tracecmd/trace-msg.c b/tracecmd/trace-msg.c
index edde582..5173165 100644
--- a/tracecmd/trace-msg.c
+++ b/tracecmd/trace-msg.c
@@ -16,6 +16,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
+#include <string.h>
#include <unistd.h>
#include <arpa/inet.h>
#include <sys/types.h>
@@ -79,6 +80,16 @@ struct tracecmd_msg_rinit {
be32 cpus;
} __attribute__((packed));
+struct tracecmd_msg_trace_req {
+ be32 flags;
+ be32 argc;
+} __attribute__((packed));
+
+struct tracecmd_msg_trace_resp {
+ be32 cpus;
+ be32 page_size;
+} __attribute__((packed));
+
struct tracecmd_msg_header {
be32 size;
be32 cmd;
@@ -90,7 +101,9 @@ struct tracecmd_msg_header {
C(TINIT, 1, sizeof(struct tracecmd_msg_tinit)), \
C(RINIT, 2, sizeof(struct tracecmd_msg_rinit)), \
C(SEND_DATA, 3, 0), \
- C(FIN_DATA, 4, 0),
+ C(FIN_DATA, 4, 0), \
+ C(TRACE_REQ, 5, sizeof(struct tracecmd_msg_trace_req)), \
+ C(TRACE_RESP, 6, sizeof(struct tracecmd_msg_trace_resp)),
#undef C
#define C(a,b,c) MSG_##a = b
@@ -122,6 +135,8 @@ struct tracecmd_msg {
union {
struct tracecmd_msg_tinit tinit;
struct tracecmd_msg_rinit rinit;
+ struct tracecmd_msg_trace_req trace_req;
+ struct tracecmd_msg_trace_resp trace_resp;
};
union {
struct tracecmd_msg_opt *opt;
@@ -715,3 +730,185 @@ error:
msg_free(&msg);
return ret;
}
+
+static int make_trace_req(struct tracecmd_msg *msg, int argc, char **argv)
+{
+ size_t args_size = 0;
+ char *p;
+ int i;
+
+ for (i = 0; i < argc; i++)
+ args_size += strlen(argv[i]) + 1;
+
+ msg->hdr.size = htonl(ntohl(msg->hdr.size) + args_size);
+ msg->trace_req.argc = htonl(argc);
+ msg->buf = calloc(args_size, 1);
+ if (!msg->buf)
+ return -ENOMEM;
+
+ p = msg->buf;
+ for (i = 0; i < argc; i++)
+ p = stpcpy(p, argv[i]) + 1;
+
+ return 0;
+}
+
+int tracecmd_msg_send_trace_req(struct tracecmd_msg_handle *msg_handle,
+ int argc, char **argv)
+{
+ struct tracecmd_msg msg;
+ int ret;
+
+ tracecmd_msg_init(MSG_TRACE_REQ, &msg);
+ ret = make_trace_req(&msg, argc, argv);
+ if (ret)
+ return ret;
+
+ return tracecmd_msg_send(msg_handle->fd, &msg);
+}
+
+ /*
+ * NOTE: On success, the returned `argv` should be freed with:
+ * free(argv[0]);
+ * free(argv);
+ */
+int tracecmd_msg_recv_trace_req(struct tracecmd_msg_handle *msg_handle,
+ int *argc, char ***argv)
+{
+ struct tracecmd_msg msg;
+ char *p, *buf_end, **args;
+ int i, ret, nr_args;
+ size_t buf_len;
+
+ ret = tracecmd_msg_recv(msg_handle->fd, &msg);
+ if (ret < 0)
+ return ret;
+
+ if (ntohl(msg.hdr.cmd) != MSG_TRACE_REQ)
+ goto out;
+
+ if (ntohl(msg.trace_req.argc) < 0)
+ goto out;
+
+ buf_len = ntohl(msg.hdr.size) - MSG_HDR_LEN - ntohl(msg.hdr.cmd_size);
+ buf_end = (char *)msg.buf + buf_len;
+ p = msg.buf;
+ nr_args = ntohl(msg.trace_req.argc);
+ args = calloc(nr_args, sizeof(*args));
+ if (!args) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ for (i = 0; i < nr_args; i++) {
+ if (p >= buf_end) {
+ ret = -1;
+ free(args);
+ goto out;
+ }
+
+ args[i] = p;
+ p = strchr(p, '\0');
+ p++;
+ }
+
+ /*
+ * On success we're passing msg.buf to the caller through argv[0] so we
+ * reset it here to avoid getting it freed below.
+ */
+ msg.buf = NULL;
+ *argc = nr_args;
+ *argv = args;
+ ret = 0;
+
+out:
+ msg_free(&msg);
+ return ret;
+}
+
+static int make_trace_resp(struct tracecmd_msg *msg,
+ int page_size, int nr_cpus, int *ports)
+{
+ int ports_size = nr_cpus * sizeof(*msg->port_array);
+ int i;
+
+ msg->hdr.size = htonl(ntohl(msg->hdr.size) + ports_size);
+ msg->trace_resp.cpus = htonl(nr_cpus);
+ msg->trace_resp.page_size = htonl(page_size);
+
+ msg->port_array = malloc(ports_size);
+ if (!msg->port_array)
+ return -ENOMEM;
+
+ for (i = 0; i < nr_cpus; i++)
+ msg->port_array[i] = htonl(ports[i]);
+
+ return 0;
+}
+
+int tracecmd_msg_send_trace_resp(struct tracecmd_msg_handle *msg_handle,
+ int nr_cpus, int page_size, int *ports)
+{
+ struct tracecmd_msg msg;
+ int ret;
+
+ tracecmd_msg_init(MSG_TRACE_RESP, &msg);
+ ret = make_trace_resp(&msg, page_size, nr_cpus, ports);
+ if (ret < 0)
+ return ret;
+
+ return tracecmd_msg_send(msg_handle->fd, &msg);
+}
+
+int tracecmd_msg_recv_trace_resp(struct tracecmd_msg_handle *msg_handle,
+ int *nr_cpus, int *page_size, int **ports)
+{
+ struct tracecmd_msg msg;
+ size_t buf_len;
+ int i, ret;
+
+ ret = tracecmd_msg_recv(msg_handle->fd, &msg);
+ if (ret < 0)
+ return ret;
+
+ if (ntohl(msg.hdr.cmd) != MSG_TRACE_RESP) {
+ ret = -1;
+ goto out;
+ }
+
+ buf_len = ntohl(msg.hdr.size) - MSG_HDR_LEN - ntohl(msg.hdr.cmd_size);
+ if (buf_len <= 0 ||
+ buf_len != sizeof(*msg.port_array) * ntohl(msg.trace_resp.cpus)) {
+ ret = -1;
+ goto out;
+ }
+
+ *nr_cpus = ntohl(msg.trace_resp.cpus);
+ *page_size = ntohl(msg.trace_resp.page_size);
+ *ports = calloc(*nr_cpus, sizeof(**ports));
+ if (!*ports) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ for (i = 0; i < *nr_cpus; i++)
+ (*ports)[i] = ntohl(msg.port_array[i]);
+
+ ret = 0;
+
+out:
+ msg_free(&msg);
+ return ret;
+}
+
+int tracecmd_msg_wait_close(struct tracecmd_msg_handle *msg_handle)
+{
+ char buf[BUFSIZ];
+ int ret;
+
+ for (;;) {
+ ret = read(msg_handle->fd, buf, sizeof(buf));
+ if (ret <= 0)
+ return ret;
+ }
+
+ return -1;
+}
--
2.19.1
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH v3 5/6] trace-cmd: Add buffer instance flags for tracing in guest and agent context
2019-01-14 15:27 [PATCH v3 0/6] Add VM kernel tracing over vsock sockets Slavomir Kaslev
` (3 preceding siblings ...)
2019-01-14 15:27 ` [PATCH v3 4/6] trace-cmd: Add TRACE_REQ and TRACE_RESP messages Slavomir Kaslev
@ 2019-01-14 15:27 ` Slavomir Kaslev
2019-01-14 15:28 ` [PATCH v3 6/6] trace-cmd: Add VM kernel tracing over vsock sockets transport Slavomir Kaslev
5 siblings, 0 replies; 12+ messages in thread
From: Slavomir Kaslev @ 2019-01-14 15:27 UTC (permalink / raw)
To: linux-trace-devel; +Cc: rostedt, ykaradzhov, tstoyanov
Add BUFFER_FL_GUEST and BUFFER_FL_AGENT flags to differentiate when
trace-record.c is being called to trace guest or the VM tracing agent.
Also disable functions talking to the local tracefs when called in recording
guest instances context.
Signed-off-by: Slavomir Kaslev <kaslevs@vmware.com>
---
tracecmd/include/trace-local.h | 2 ++
tracecmd/trace-record.c | 55 ++++++++++++++++++++++++++++++++--
2 files changed, 55 insertions(+), 2 deletions(-)
diff --git a/tracecmd/include/trace-local.h b/tracecmd/include/trace-local.h
index a1a06e9..f19c8bb 100644
--- a/tracecmd/include/trace-local.h
+++ b/tracecmd/include/trace-local.h
@@ -149,6 +149,8 @@ char *strstrip(char *str);
enum buffer_instance_flags {
BUFFER_FL_KEEP = 1 << 0,
BUFFER_FL_PROFILE = 1 << 1,
+ BUFFER_FL_GUEST = 1 << 2,
+ BUFFER_FL_AGENT = 1 << 3,
};
struct func_list {
diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index a8c3464..012371c 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -781,6 +781,9 @@ static void __clear_trace(struct buffer_instance *instance)
FILE *fp;
char *path;
+ if (instance->flags & BUFFER_FL_GUEST)
+ return;
+
/* reset the trace */
path = get_instance_file(instance, "trace");
fp = fopen(path, "w");
@@ -1264,6 +1267,9 @@ set_plugin_instance(struct buffer_instance *instance, const char *name)
char *path;
char zero = '0';
+ if (instance->flags & BUFFER_FL_GUEST)
+ return;
+
path = get_instance_file(instance, "current_tracer");
fp = fopen(path, "w");
if (!fp) {
@@ -1360,6 +1366,9 @@ static void disable_func_stack_trace_instance(struct buffer_instance *instance)
int size;
int ret;
+ if (instance->flags & BUFFER_FL_GUEST)
+ return;
+
path = get_instance_file(instance, "current_tracer");
ret = stat(path, &st);
tracecmd_put_tracing_file(path);
@@ -1553,6 +1562,9 @@ reset_events_instance(struct buffer_instance *instance)
int i;
int ret;
+ if (instance->flags & BUFFER_FL_GUEST)
+ return;
+
if (use_old_event_method()) {
/* old way only had top instance */
if (!is_top_instance(instance))
@@ -1904,6 +1916,9 @@ static void write_tracing_on(struct buffer_instance *instance, int on)
int ret;
int fd;
+ if (instance->flags & BUFFER_FL_GUEST)
+ return;
+
fd = open_tracing_on(instance);
if (fd < 0)
return;
@@ -1923,6 +1938,9 @@ static int read_tracing_on(struct buffer_instance *instance)
char buf[10];
int ret;
+ if (instance->flags & BUFFER_FL_GUEST)
+ return -1;
+
fd = open_tracing_on(instance);
if (fd < 0)
return fd;
@@ -2156,6 +2174,9 @@ static void set_mask(struct buffer_instance *instance)
int fd;
int ret;
+ if (instance->flags & BUFFER_FL_GUEST)
+ return;
+
if (!instance->cpumask)
return;
@@ -2186,6 +2207,9 @@ static void enable_events(struct buffer_instance *instance)
{
struct event_list *event;
+ if (instance->flags & BUFFER_FL_GUEST)
+ return;
+
for (event = instance->events; event; event = event->next) {
if (!event->neg)
update_event(event, event->filter, 0, '1');
@@ -2209,6 +2233,9 @@ static void set_clock(struct buffer_instance *instance)
char *content;
char *str;
+ if (instance->flags & BUFFER_FL_GUEST)
+ return;
+
if (!instance->clock)
return;
@@ -2238,6 +2265,9 @@ static void set_max_graph_depth(struct buffer_instance *instance, char *max_grap
char *path;
int ret;
+ if (instance->flags & BUFFER_FL_GUEST)
+ return;
+
path = get_instance_file(instance, "max_graph_depth");
reset_save_file(path, RESET_DEFAULT_PRIO);
tracecmd_put_tracing_file(path);
@@ -2463,6 +2493,9 @@ static void expand_event_instance(struct buffer_instance *instance)
struct event_list *compressed_list = instance->events;
struct event_list *event;
+ if (instance->flags & BUFFER_FL_GUEST)
+ return;
+
reset_event_list(instance);
while (compressed_list) {
@@ -3336,6 +3369,9 @@ static void set_funcs(struct buffer_instance *instance)
int set_notrace = 0;
int ret;
+ if (instance->flags & BUFFER_FL_GUEST)
+ return;
+
ret = write_func_file(instance, "set_ftrace_filter", &instance->filter_funcs);
if (ret < 0)
die("set_ftrace_filter does not exist. Can not filter functions");
@@ -3632,6 +3668,9 @@ static void set_buffer_size_instance(struct buffer_instance *instance)
int ret;
int fd;
+ if (instance->flags & BUFFER_FL_GUEST)
+ return;
+
if (!buffer_size)
return;
@@ -3829,6 +3868,9 @@ static void make_instances(void)
int ret;
for_each_instance(instance) {
+ if (instance->flags & BUFFER_FL_GUEST)
+ continue;
+
path = get_instance_dir(instance);
ret = stat(path, &st);
if (ret < 0) {
@@ -3850,7 +3892,7 @@ void tracecmd_remove_instances(void)
for_each_instance(instance) {
/* Only delete what we created */
- if (instance->flags & BUFFER_FL_KEEP)
+ if (instance->flags & (BUFFER_FL_KEEP | BUFFER_FL_GUEST))
continue;
if (instance->tracing_on_fd > 0) {
close(instance->tracing_on_fd);
@@ -3932,7 +3974,7 @@ static void check_function_plugin(void)
static int __check_doing_something(struct buffer_instance *instance)
{
- return (instance->flags & BUFFER_FL_PROFILE) ||
+ return (instance->flags & (BUFFER_FL_PROFILE | BUFFER_FL_GUEST)) ||
instance->plugin || instance->events;
}
@@ -3954,6 +3996,9 @@ update_plugin_instance(struct buffer_instance *instance,
{
const char *plugin = instance->plugin;
+ if (instance->flags & BUFFER_FL_GUEST)
+ return;
+
if (!plugin)
return;
@@ -4053,6 +4098,9 @@ static void record_stats(void)
int cpu;
for_all_instances(instance) {
+ if (instance->flags & BUFFER_FL_GUEST)
+ continue;
+
s_save = instance->s_save;
s_print = instance->s_print;
for (cpu = 0; cpu < instance->cpu_count; cpu++) {
@@ -4079,6 +4127,9 @@ static void destroy_stats(void)
int cpu;
for_all_instances(instance) {
+ if (instance->flags & BUFFER_FL_GUEST)
+ continue;
+
for (cpu = 0; cpu < instance->cpu_count; cpu++) {
trace_seq_destroy(&instance->s_save[cpu]);
trace_seq_destroy(&instance->s_print[cpu]);
--
2.19.1
^ permalink raw reply related [flat|nested] 12+ messages in thread
* [PATCH v3 6/6] trace-cmd: Add VM kernel tracing over vsock sockets transport
2019-01-14 15:27 [PATCH v3 0/6] Add VM kernel tracing over vsock sockets Slavomir Kaslev
` (4 preceding siblings ...)
2019-01-14 15:27 ` [PATCH v3 5/6] trace-cmd: Add buffer instance flags for tracing in guest and agent context Slavomir Kaslev
@ 2019-01-14 15:28 ` Slavomir Kaslev
2019-01-14 22:46 ` Steven Rostedt
5 siblings, 1 reply; 12+ messages in thread
From: Slavomir Kaslev @ 2019-01-14 15:28 UTC (permalink / raw)
To: linux-trace-devel; +Cc: rostedt, ykaradzhov, tstoyanov
To test, start a VM and assign it a valid (> 2) unused CID:
you@host # qemu-system-x86_64 \ -name guest2 --uuid
7edfdf4a-cb9e-11e8-b38a-173b58342476 \ -m 4096 -boot d -enable-kvm -smp 3
-net nic -net user -hda $HOME/vm/guest2.img \ -device
vhost-vsock-pci,id=vhost-vsock-pci0,guest-cid=3
and start `trace-cmd agent` on the guest as root:
you@guest2 # trace-cmd agent
Finally, start `trace-cmd record` on the host (running as root is only necessary
if the host will be traced too):
you@host $ trace-cmd record -A guest2 -e irq -e sched
Signed-off-by: Slavomir Kaslev <kaslevs@vmware.com>
---
tracecmd/Makefile | 6 +-
tracecmd/include/trace-local.h | 17 +
tracecmd/trace-agent.c | 229 ++++++++++++
tracecmd/trace-cmd.c | 3 +
tracecmd/trace-record.c | 613 ++++++++++++++++++++++++++++++---
tracecmd/trace-usage.c | 13 +-
6 files changed, 829 insertions(+), 52 deletions(-)
create mode 100644 tracecmd/trace-agent.c
diff --git a/tracecmd/Makefile b/tracecmd/Makefile
index 3a11024..865b1c6 100644
--- a/tracecmd/Makefile
+++ b/tracecmd/Makefile
@@ -33,13 +33,17 @@ TRACE_CMD_OBJS += trace-output.o
TRACE_CMD_OBJS += trace-usage.o
TRACE_CMD_OBJS += trace-msg.o
+ifeq ($(VSOCK_DEFINED), 1)
+TRACE_CMD_OBJS += trace-agent.o
+endif
+
ALL_OBJS := $(TRACE_CMD_OBJS:%.o=$(bdir)/%.o)
all_objs := $(sort $(ALL_OBJS))
all_deps := $(all_objs:$(bdir)/%.o=$(bdir)/.%.d)
CONFIG_INCLUDES =
-CONFIG_LIBS =
+CONFIG_LIBS = -lrt
CONFIG_FLAGS =
all: $(TARGETS)
diff --git a/tracecmd/include/trace-local.h b/tracecmd/include/trace-local.h
index f19c8bb..5072d8e 100644
--- a/tracecmd/include/trace-local.h
+++ b/tracecmd/include/trace-local.h
@@ -12,6 +12,8 @@
#include "trace-cmd.h"
#include "event-utils.h"
+#define TRACE_AGENT_DEFAULT_PORT 823
+
extern int debug;
extern int quiet;
@@ -64,6 +66,8 @@ void trace_split(int argc, char **argv);
void trace_listen(int argc, char **argv);
+void trace_agent(int argc, char **argv);
+
void trace_restore(int argc, char **argv);
void trace_clear(int argc, char **argv);
@@ -88,6 +92,10 @@ void trace_list(int argc, char **argv);
void trace_usage(int argc, char **argv);
+int trace_record_agent(struct tracecmd_msg_handle *msg_handle,
+ int cpus, int *fds,
+ int argc, char **argv);
+
struct hook_list;
void trace_init_profile(struct tracecmd_input *handle, struct hook_list *hooks,
@@ -176,6 +184,7 @@ struct buffer_instance {
struct func_list *notrace_funcs;
const char *clock;
+ int *client_ports;
struct trace_seq *s_save;
struct trace_seq *s_print;
@@ -190,6 +199,14 @@ struct buffer_instance {
int tracing_on_fd;
int buffer_size;
int cpu_count;
+
+ int argc;
+ char **argv;
+ int argv_cap;
+
+ int cid;
+ int port;
+ int *fds;
};
extern struct buffer_instance top_instance;
diff --git a/tracecmd/trace-agent.c b/tracecmd/trace-agent.c
new file mode 100644
index 0000000..b5e812a
--- /dev/null
+++ b/tracecmd/trace-agent.c
@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: LGPL-2.1
+/*
+ * Copyright (C) 2018 VMware Inc, Slavomir Kaslev <kaslevs@vmware.com>
+ *
+ * based on prior implementation by Yoshihiro Yunomae
+ * Copyright (C) 2013 Hitachi, Ltd.
+ * Yoshihiro YUNOMAE <yoshihiro.yunomae.ez@hitachi.com>
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <linux/vm_sockets.h>
+
+#include "trace-local.h"
+#include "trace-msg.h"
+
+static int make_vsock(unsigned port)
+{
+ struct sockaddr_vm addr = {
+ .svm_family = AF_VSOCK,
+ .svm_cid = VMADDR_CID_ANY,
+ .svm_port = port,
+ };
+ int sd;
+
+ sd = socket(AF_VSOCK, SOCK_STREAM, 0);
+ if (sd < 0)
+ return -1;
+
+ setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &(int){1}, sizeof(int));
+
+ if (bind(sd, (struct sockaddr *)&addr, sizeof(addr)))
+ return -1;
+
+ if (listen(sd, SOMAXCONN))
+ return -1;
+
+ return sd;
+}
+
+static int get_vsock_port(int sd)
+{
+ struct sockaddr_vm addr;
+ socklen_t addr_len = sizeof(addr);
+
+ if (getsockname(sd, (struct sockaddr *)&addr, &addr_len))
+ return -1;
+
+ if (addr.svm_family != AF_VSOCK)
+ return -1;
+
+ return addr.svm_port;
+}
+
+static void make_vsocks(int nr, int **fds, int **ports)
+{
+ int i, fd, port;
+
+ *fds = calloc(nr, sizeof(*fds));
+ *ports = calloc(nr, sizeof(*ports));
+ if (!*fds || !*ports)
+ die("Failed to allocate memory");
+
+ for (i = 0; i < nr; i++) {
+ fd = make_vsock(VMADDR_PORT_ANY);
+ if (fd < 0)
+ die("Failed to open vsock socket");
+
+ port = get_vsock_port(fd);
+ if (port < 0)
+ die("Failed to get vsock socket address");
+
+ (*fds)[i] = fd;
+ (*ports)[i] = port;
+ }
+}
+
+static void free_vsocks(int nr, int *fds, int *ports)
+{
+ int i;
+
+ for (i = 0; i < nr; i++)
+ close(fds[i]);
+ free(fds);
+ free(ports);
+}
+
+static void agent_handle(int sd, int nr_cpus, int page_size)
+{
+ struct tracecmd_msg_handle *msg_handle;
+ int *fds, *ports;
+ char **argv = NULL;
+ int argc = 0;
+
+ msg_handle = tracecmd_msg_handle_alloc(sd, TRACECMD_MSG_FL_CLIENT);
+ if (!msg_handle)
+ die("Failed to allocate message handle");
+
+ if (tracecmd_msg_recv_trace_req(msg_handle, &argc, &argv))
+ die("Failed to receive trace request");
+
+ make_vsocks(nr_cpus, &fds, &ports);
+
+ if (tracecmd_msg_send_trace_resp(msg_handle, nr_cpus, page_size, ports))
+ die("Failed to send trace response");
+
+ trace_record_agent(msg_handle, nr_cpus, fds, argc, argv);
+
+ free_vsocks(nr_cpus, fds, ports);
+ free(argv[0]);
+ free(argv);
+ tracecmd_msg_handle_close(msg_handle);
+ exit(0);
+}
+
+static volatile pid_t handler_pid;
+
+static void handle_sigchld(int sig)
+{
+ int wstatus;
+ pid_t pid;
+
+ for (;;) {
+ pid = waitpid(-1, &wstatus, WNOHANG);
+ if (pid <= 0)
+ break;
+
+ if (pid == handler_pid)
+ handler_pid = 0;
+ }
+}
+
+static void agent_serve(unsigned port)
+{
+ int sd, cd, nr_cpus;
+ pid_t pid;
+
+ signal(SIGCHLD, handle_sigchld);
+
+ nr_cpus = count_cpus();
+ page_size = getpagesize();
+
+ sd = make_vsock(port);
+ if (sd < 0)
+ die("Failed to open vsock socket");
+
+ for (;;) {
+ cd = accept(sd, NULL, NULL);
+ if (cd < 0) {
+ if (errno == EINTR)
+ continue;
+ die("accept");
+ }
+
+ if (handler_pid)
+ goto busy;
+
+ pid = fork();
+ if (pid == 0) {
+ close(sd);
+ signal(SIGCHLD, SIG_DFL);
+ agent_handle(cd, nr_cpus, page_size);
+ }
+ if (pid > 0)
+ handler_pid = pid;
+
+ busy:
+ close(cd);
+ }
+
+ close(sd);
+ signal(SIGCHLD, SIG_DFL);
+}
+
+void trace_agent(int argc, char **argv)
+{
+ bool do_daemon = false;
+ unsigned port = TRACE_AGENT_DEFAULT_PORT;
+
+ if (argc < 2)
+ usage(argv);
+
+ if (strcmp(argv[1], "agent") != 0)
+ usage(argv);
+
+ for (;;) {
+ int c, option_index = 0;
+ static struct option long_options[] = {
+ {"port", required_argument, NULL, 'p'},
+ {"help", no_argument, NULL, '?'},
+ {NULL, 0, NULL, 0}
+ };
+
+ c = getopt_long(argc-1, argv+1, "+hp:D",
+ long_options, &option_index);
+ if (c == -1)
+ break;
+ switch (c) {
+ case 'h':
+ usage(argv);
+ break;
+ case 'p':
+ port = atoi(optarg);
+ break;
+ case 'D':
+ do_daemon = true;
+ break;
+ default:
+ usage(argv);
+ }
+ }
+
+ if ((argc - optind) >= 2)
+ usage(argv);
+
+ if (do_daemon && daemon(1, 0))
+ die("daemon");
+
+ agent_serve(port);
+}
diff --git a/tracecmd/trace-cmd.c b/tracecmd/trace-cmd.c
index 797b303..3ae5e2e 100644
--- a/tracecmd/trace-cmd.c
+++ b/tracecmd/trace-cmd.c
@@ -83,6 +83,9 @@ struct command commands[] = {
{"hist", trace_hist},
{"mem", trace_mem},
{"listen", trace_listen},
+#ifdef VSOCK
+ {"agent", trace_agent},
+#endif
{"split", trace_split},
{"restore", trace_restore},
{"stack", trace_stack},
diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index 012371c..6c8754e 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -33,6 +33,9 @@
#include <errno.h>
#include <limits.h>
#include <libgen.h>
+#ifdef VSOCK
+#include <linux/vm_sockets.h>
+#endif
#include "trace-local.h"
#include "trace-msg.h"
@@ -74,8 +77,6 @@ static int buffers;
static int clear_function_filters;
static char *host;
-static int *client_ports;
-static int sfd;
/* Max size to let a per cpu file get */
static int max_kb;
@@ -171,6 +172,15 @@ static struct tracecmd_recorder *recorder;
static int ignore_event_not_found = 0;
+static inline size_t grow_cap(size_t old_cap)
+{
+ size_t cap = 3 * old_cap / 2;
+
+ if (cap < 16)
+ cap = 16;
+ return cap;
+}
+
static inline int is_top_instance(struct buffer_instance *instance)
{
return instance == &top_instance;
@@ -518,6 +528,36 @@ static char *get_temp_file(struct buffer_instance *instance, int cpu)
return file;
}
+static char *get_guest_file(const char *file, const char *guest)
+{
+ size_t guest_len = strlen(guest);
+ size_t file_len = strlen(file);
+ size_t base_len, idx = 0;
+ const char *p;
+ char *out;
+
+ out = malloc(file_len + guest_len + 2 /* dash and \0 */);
+ if (!out)
+ return NULL;
+
+ p = strrchr(file, '.');
+ if (p && p != file)
+ base_len = p - file;
+ else
+ base_len = file_len;
+
+ memcpy(out, file, base_len);
+ idx += base_len;
+ out[idx++] = '-';
+ memcpy(out + idx, guest, guest_len);
+ idx += guest_len;
+ memcpy(out + idx, p, file_len - base_len);
+ idx += file_len - base_len;
+ out[idx] = '\0';
+
+ return out;
+}
+
static void put_temp_file(char *file)
{
free(file);
@@ -623,6 +663,16 @@ static void delete_thread_data(void)
}
}
+static void tell_guests_to_stop(void)
+{
+ struct buffer_instance *instance;
+
+ for_all_instances(instance) {
+ if (instance->flags & BUFFER_FL_GUEST)
+ tracecmd_msg_handle_close(instance->msg_handle);
+ }
+}
+
static void stop_threads(enum trace_type type)
{
struct timeval tv = { 0, 0 };
@@ -632,6 +682,8 @@ static void stop_threads(enum trace_type type)
if (!recorder_threads)
return;
+ tell_guests_to_stop();
+
/* Tell all threads to finish up */
for (i = 0; i < recorder_threads; i++) {
if (pids[i].pid > 0) {
@@ -2571,14 +2623,14 @@ static void flush(int sig)
tracecmd_stop_recording(recorder);
}
-static void connect_port(int cpu)
+static int connect_port(const char *host, unsigned port)
{
struct addrinfo hints;
struct addrinfo *results, *rp;
- int s;
+ int s, sfd;
char buf[BUFSIZ];
- snprintf(buf, BUFSIZ, "%d", client_ports[cpu]);
+ snprintf(buf, BUFSIZ, "%d", port);
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
@@ -2605,7 +2657,190 @@ static void connect_port(int cpu)
freeaddrinfo(results);
- client_ports[cpu] = sfd;
+ return sfd;
+}
+
+#ifdef VSOCK
+static int open_vsock(unsigned cid, unsigned port)
+{
+ struct sockaddr_vm addr = {
+ .svm_family = AF_VSOCK,
+ .svm_cid = cid,
+ .svm_port = port,
+ };
+ int sd;
+
+ sd = socket(AF_VSOCK, SOCK_STREAM, 0);
+ if (sd < 0)
+ return -1;
+
+ if (connect(sd, (struct sockaddr *)&addr, sizeof(addr)))
+ return -1;
+
+ return sd;
+}
+#else
+static inline int open_vsock(unsigned cid, unsigned port)
+{
+ die("vsock is not supported");
+ return -1;
+}
+#endif
+
+static int do_accept(int sd)
+{
+ int cd;
+
+ for (;;) {
+ cd = accept(sd, NULL, NULL);
+ if (cd < 0) {
+ if (errno == EINTR)
+ continue;
+ die("accept");
+ }
+
+ return cd;
+ }
+
+ return -1;
+}
+
+static bool is_digits(const char *s)
+{
+ const char *p;
+
+ for (p = s; *p; p++)
+ if (!isdigit(*p))
+ return false;
+
+ return true;
+}
+
+struct guest {
+ char *name;
+ int cid;
+ int pid;
+};
+
+static size_t guests_cap, guests_len;
+static struct guest *guests;
+
+static char *get_qemu_guest_name(char *arg)
+{
+ char *tok, *end = arg;
+
+ while ((tok = strsep(&end, ","))) {
+ if (strncmp(tok, "guest=", 6) == 0)
+ return tok + 6;
+ }
+
+ return arg;
+}
+
+static void read_qemu_guests(void)
+{
+ static bool initialized = false;
+ struct dirent *entry;
+ char path[PATH_MAX];
+ DIR *dir;
+
+ if (initialized)
+ return;
+
+ initialized = true;
+ dir = opendir("/proc");
+ if (!dir)
+ die("opendir");
+
+ for (entry = readdir(dir); entry; entry = readdir(dir)) {
+ bool is_qemu = false, last_was_name = false;
+ struct guest guest = {};
+ char *p, *arg = NULL;
+ size_t arg_size = 0;
+ FILE *f;
+
+ if (!(entry->d_type == DT_DIR && is_digits(entry->d_name)))
+ continue;
+
+ guest.pid = atoi(entry->d_name);
+ snprintf(path, sizeof(path), "/proc/%s/cmdline", entry->d_name);
+ f = fopen(path, "r");
+ if (!f)
+ continue;
+
+ while (getdelim(&arg, &arg_size, 0, f) != -1) {
+ if (!is_qemu && strstr(arg, "qemu-system-")) {
+ is_qemu = true;
+ continue;
+ }
+
+ if (!is_qemu)
+ continue;
+
+ if (strcmp(arg, "-name") == 0) {
+ last_was_name = true;
+ continue;
+ }
+
+ if (last_was_name) {
+ guest.name = strdup(get_qemu_guest_name(arg));
+ last_was_name = false;
+ continue;
+ }
+
+ p = strstr(arg, "guest-cid=");
+ if (p) {
+ guest.cid = atoi(p + 10);
+ continue;
+ }
+ }
+
+ if (is_qemu) {
+ if (guests_cap == guests_len) {
+ guests_cap = grow_cap(guests_cap);
+ guests = realloc(guests,
+ guests_cap * sizeof(*guests));
+ }
+ guests[guests_len++] = guest;
+ }
+
+ free(arg);
+ fclose(f);
+ }
+
+ closedir(dir);
+}
+
+static char *parse_guest_name(char *guest, int *cid, int *port)
+{
+ size_t i;
+ char *p;
+
+ *port = -1;
+ p = strrchr(guest, ':');
+ if (p) {
+ *p = '\0';
+ *port = atoi(p + 1);
+ }
+
+ *cid = -1;
+ p = strrchr(guest, '@');
+ if (p) {
+ *p = '\0';
+ *cid = atoi(p + 1);
+ } else if (is_digits(guest))
+ *cid = atoi(guest);
+
+ read_qemu_guests();
+ for (i = 0; i < guests_len; i++) {
+ if ((*cid > 0 && *cid == guests[i].cid) ||
+ strcmp(guest, guests[i].name) == 0) {
+ *cid = guests[i].cid;
+ return guests[i].name;
+ }
+ }
+
+ return guest;
}
static void set_prio(int prio)
@@ -2652,6 +2887,16 @@ create_recorder_instance(struct buffer_instance *instance, const char *file, int
struct tracecmd_recorder *record;
char *path;
+ if (instance->flags & BUFFER_FL_GUEST) {
+ int sd;
+
+ sd = open_vsock(instance->cid, instance->client_ports[cpu]);
+ if (sd < 0)
+ die("Failed to connect to agent");
+
+ return tracecmd_create_recorder_virt(file, cpu, sd);
+ }
+
if (brass)
return create_recorder_instance_pipe(instance, cpu, brass);
@@ -2676,7 +2921,7 @@ static int create_recorder(struct buffer_instance *instance, int cpu,
{
long ret;
char *file;
- int pid;
+ pid_t pid;
if (type != TRACE_TYPE_EXTRACT) {
signal(SIGUSR1, flush);
@@ -2695,19 +2940,26 @@ static int create_recorder(struct buffer_instance *instance, int cpu,
instance->cpu_count = 0;
}
- if (client_ports) {
- char *path;
+ if ((instance->client_ports && !(instance->flags & BUFFER_FL_GUEST)) ||
+ (instance->flags & BUFFER_FL_AGENT)) {
+ unsigned flags = recorder_flags;
+ char *path = NULL;
+ int fd;
- connect_port(cpu);
- if (instance->name)
+ if (instance->flags & BUFFER_FL_AGENT) {
+ fd = do_accept(instance->fds[cpu]);
+ flags |= TRACECMD_RECORD_NOSPLICE;
+ } else {
+ fd = connect_port(host, instance->client_ports[cpu]);
+ }
+ if (fd < 0)
+ die("Failed connecting to client");
+ if (instance->name && !(instance->flags & BUFFER_FL_AGENT))
path = get_instance_dir(instance);
else
path = tracecmd_find_tracing_dir();
- recorder = tracecmd_create_buffer_recorder_fd(client_ports[cpu],
- cpu, recorder_flags,
- path);
- if (instance->name)
- tracecmd_put_tracing_file(path);
+ recorder = tracecmd_create_buffer_recorder_fd(fd, cpu, flags, path);
+ tracecmd_put_tracing_file(path);
} else {
file = get_temp_file(instance, cpu);
recorder = create_recorder_instance(instance, file, cpu, brass);
@@ -2745,7 +2997,8 @@ static void check_first_msg_from_server(struct tracecmd_msg_handle *msg_handle)
die("server not tracecmd server");
}
-static void communicate_with_listener_v1(struct tracecmd_msg_handle *msg_handle)
+static void communicate_with_listener_v1(struct tracecmd_msg_handle *msg_handle,
+ int **client_ports)
{
char buf[BUFSIZ];
ssize_t n;
@@ -2788,8 +3041,8 @@ static void communicate_with_listener_v1(struct tracecmd_msg_handle *msg_handle)
/* No options */
write(msg_handle->fd, "0", 2);
- client_ports = malloc(sizeof(int) * local_cpu_count);
- if (!client_ports)
+ *client_ports = malloc(sizeof(int) * local_cpu_count);
+ if (!*client_ports)
die("Failed to allocate client ports for %d cpus", local_cpu_count);
/*
@@ -2807,13 +3060,14 @@ static void communicate_with_listener_v1(struct tracecmd_msg_handle *msg_handle)
if (i == BUFSIZ)
die("read bad port number");
buf[i] = 0;
- client_ports[cpu] = atoi(buf);
+ (*client_ports)[cpu] = atoi(buf);
}
}
-static void communicate_with_listener_v3(struct tracecmd_msg_handle *msg_handle)
+static void communicate_with_listener_v3(struct tracecmd_msg_handle *msg_handle,
+ int **client_ports)
{
- if (tracecmd_msg_send_init_data(msg_handle, &client_ports) < 0)
+ if (tracecmd_msg_send_init_data(msg_handle, client_ports) < 0)
die("Cannot communicate with server");
}
@@ -2864,7 +3118,7 @@ static void check_protocol_version(struct tracecmd_msg_handle *msg_handle)
}
}
-static struct tracecmd_msg_handle *setup_network(void)
+static struct tracecmd_msg_handle *setup_network(struct buffer_instance *instance)
{
struct tracecmd_msg_handle *msg_handle = NULL;
struct addrinfo hints;
@@ -2934,11 +3188,11 @@ again:
close(sfd);
goto again;
}
- communicate_with_listener_v3(msg_handle);
+ communicate_with_listener_v3(msg_handle, &instance->client_ports);
}
if (msg_handle->version == V1_PROTOCOL)
- communicate_with_listener_v1(msg_handle);
+ communicate_with_listener_v1(msg_handle, &instance->client_ports);
return msg_handle;
}
@@ -2951,7 +3205,7 @@ setup_connection(struct buffer_instance *instance, struct common_record_context
struct tracecmd_msg_handle *msg_handle;
struct tracecmd_output *network_handle;
- msg_handle = setup_network();
+ msg_handle = setup_network(instance);
/* Now create the handle through this socket */
if (msg_handle->version == V3_PROTOCOL) {
@@ -2978,28 +3232,96 @@ static void finish_network(struct tracecmd_msg_handle *msg_handle)
free(host);
}
+static void connect_to_agent(struct buffer_instance *instance)
+{
+ struct tracecmd_msg_handle *msg_handle;
+ int sd, nr_cpus, page_size, *ports;
+
+ sd = open_vsock(instance->cid, instance->port);
+ if (sd < 0)
+ die("Failed to connect to vsock socket @%d:%d",
+ instance->cid, instance->port);
+
+ msg_handle = tracecmd_msg_handle_alloc(sd, TRACECMD_MSG_FL_SERVER);
+ if (!msg_handle)
+ die("Failed to allocate message handle");
+
+ if (tracecmd_msg_send_trace_req(msg_handle, instance->argc, instance->argv))
+ die("Failed to send trace request");
+
+ if (tracecmd_msg_recv_trace_resp(msg_handle, &nr_cpus, &page_size, &ports))
+ die("Failed to receive trace response");
+
+ instance->client_ports = ports;
+ instance->cpu_count = nr_cpus;
+
+ /* the msg_handle now points to the guest fd */
+ instance->msg_handle = msg_handle;
+}
+
+static void setup_guest(struct buffer_instance *instance)
+{
+ struct tracecmd_msg_handle *msg_handle = instance->msg_handle;
+ char *file;
+ int fd;
+
+ /* Create a place to store the guest meta data */
+ file = get_guest_file(output_file, instance->name);
+ if (!file)
+ die("Failed to allocate memory");
+
+ fd = open(file, O_CREAT|O_WRONLY|O_TRUNC, 0644);
+ put_temp_file(file);
+ if (fd < 0)
+ die("Failed to open", file);
+
+ /* Start reading the fds here */
+ if (tracecmd_msg_read_data(msg_handle, fd))
+ die("Failed receiving metadata");
+ close(fd);
+}
+
+static void setup_agent(struct buffer_instance *instance, struct common_record_context *ctx)
+{
+ struct tracecmd_output *network_handle;
+
+ network_handle = tracecmd_create_init_fd_msg(instance->msg_handle,
+ listed_events);
+ add_options(network_handle, ctx);
+ tracecmd_write_cpus(network_handle, instance->cpu_count);
+ tracecmd_write_options(network_handle);
+ tracecmd_msg_finish_sending_data(instance->msg_handle);
+ instance->network_handle = network_handle;
+}
+
void start_threads(enum trace_type type, struct common_record_context *ctx)
{
struct buffer_instance *instance;
- int *brass = NULL;
int total_cpu_count = 0;
int i = 0;
int ret;
- for_all_instances(instance)
+ for_all_instances(instance) {
+ /* Start the connection now to find out how many CPUs we need */
+ if (instance->flags & BUFFER_FL_GUEST)
+ connect_to_agent(instance);
total_cpu_count += instance->cpu_count;
+ }
/* make a thread for every CPU we have */
- pids = malloc(sizeof(*pids) * total_cpu_count * (buffers + 1));
+ pids = calloc(total_cpu_count * (buffers + 1), sizeof(*pids));
if (!pids)
- die("Failed to allocat pids for %d cpus", total_cpu_count);
-
- memset(pids, 0, sizeof(*pids) * total_cpu_count * (buffers + 1));
+ die("Failed to allocate pids for %d cpus", total_cpu_count);
for_all_instances(instance) {
+ int *brass = NULL;
int x, pid;
- if (host) {
+ if (instance->flags & BUFFER_FL_AGENT) {
+ setup_agent(instance, ctx);
+ } else if (instance->flags & BUFFER_FL_GUEST) {
+ setup_guest(instance);
+ } else if (host) {
instance->msg_handle = setup_connection(instance, ctx);
if (!instance->msg_handle)
die("Failed to make connection");
@@ -3139,13 +3461,14 @@ static void print_stat(struct buffer_instance *instance)
{
int cpu;
+ if (quiet)
+ return;
+
if (!is_top_instance(instance))
- if (!quiet)
- printf("\nBuffer: %s\n\n", instance->name);
+ printf("\nBuffer: %s\n\n", instance->name);
for (cpu = 0; cpu < instance->cpu_count; cpu++)
- if (!quiet)
- trace_seq_do_printf(&instance->s_print[cpu]);
+ trace_seq_do_printf(&instance->s_print[cpu]);
}
enum {
@@ -3171,7 +3494,44 @@ static void add_options(struct tracecmd_output *handle, struct common_record_con
tracecmd_add_option(handle, TRACECMD_OPTION_TRACECLOCK, 0, NULL);
add_option_hooks(handle);
add_uname(handle);
+}
+
+static void write_guest_file(struct buffer_instance *instance)
+{
+ struct tracecmd_output *handle;
+ int cpu_count = instance->cpu_count;
+ char *file;
+ char **temp_files;
+ int i, fd;
+
+ file = get_guest_file(output_file, instance->name);
+ fd = open(file, O_RDWR);
+ if (fd < 0)
+ die("error opening %s", file);
+ put_temp_file(file);
+
+ handle = tracecmd_get_output_handle_fd(fd);
+ if (!handle)
+ die("error writing to %s", file);
+
+ temp_files = malloc(sizeof(*temp_files) * cpu_count);
+ if (!temp_files)
+ die("failed to allocate temp_files for %d cpus",
+ cpu_count);
+ for (i = 0; i < cpu_count; i++) {
+ temp_files[i] = get_temp_file(instance, i);
+ if (!temp_files[i])
+ die("failed to allocate memory");
+ }
+
+ if (tracecmd_write_cpu_data(handle, cpu_count, temp_files) < 0)
+ die("failed to write CPU data");
+ tracecmd_output_close(handle);
+
+ for (i = 0; i < cpu_count; i++)
+ put_temp_file(temp_files[i]);
+ free(temp_files);
}
static void record_data(struct common_record_context *ctx)
@@ -3185,7 +3545,9 @@ static void record_data(struct common_record_context *ctx)
int i;
for_all_instances(instance) {
- if (instance->msg_handle)
+ if (instance->flags & BUFFER_FL_GUEST)
+ write_guest_file(instance);
+ else if (host && instance->msg_handle)
finish_network(instance->msg_handle);
else
local = true;
@@ -4404,6 +4766,7 @@ void trace_stop(int argc, char **argv)
c = getopt(argc-1, argv+1, "hatB:");
if (c == -1)
break;
+
switch (c) {
case 'h':
usage(argv);
@@ -4566,6 +4929,66 @@ static void init_common_record_context(struct common_record_context *ctx,
#define IS_STREAM(ctx) ((ctx)->curr_cmd == CMD_stream)
#define IS_PROFILE(ctx) ((ctx)->curr_cmd == CMD_profile)
#define IS_RECORD(ctx) ((ctx)->curr_cmd == CMD_record)
+#define IS_AGENT(ctx) ((ctx)->curr_cmd == CMD_record_agent)
+
+static void add_argv(struct buffer_instance *instance, char *arg, bool prepend)
+{
+ if (instance->argv_cap == instance->argc) {
+ instance->argv_cap = grow_cap(instance->argv_cap);
+ instance->argv = realloc(instance->argv,
+ instance->argv_cap * sizeof(char *));
+ if (!instance->argv)
+ die("Can not allocate buffer args");
+ }
+ if (prepend) {
+ memmove(instance->argv + 1, instance->argv,
+ instance->argc * sizeof(*instance->argv));
+ instance->argv[0] = arg;
+ } else {
+ instance->argv[instance->argc] = arg;
+ }
+ instance->argc++;
+}
+
+static void add_arg(struct buffer_instance *instance,
+ int c, const char *opts,
+ struct option *long_options, char *optarg)
+{
+ char *ptr;
+ char *arg;
+ int ret;
+ int i;
+
+ /* Short or long arg */
+ if (!(c & 0x80)) {
+ ret = asprintf(&arg, "-%c", c);
+ if (ret < 0)
+ die("Can not allocate argument");
+ ptr = strstr(opts, arg+1);
+ if (!ptr)
+ return; /* Not found? */
+ add_argv(instance, arg, false);
+ if (ptr[1] == ':')
+ add_argv(instance, optarg, false);
+ return;
+ }
+ for (i = 0; long_options[i].name; i++) {
+ if (c == long_options[i].val) {
+ ret = asprintf(&arg, "--%s", long_options[i].name);
+ if (ret < 0)
+ die("Can not allocate argument");
+ add_argv(instance, arg, false);
+ if (long_options[i].has_arg) {
+ arg = strdup(optarg);
+ if (!arg)
+ die("Can not allocate arguments");
+ add_argv(instance, arg, false);
+ return;
+ }
+ }
+ }
+ /* Not found? */
+}
static void parse_record_options(int argc,
char **argv,
@@ -4607,10 +5030,20 @@ static void parse_record_options(int argc,
if (IS_EXTRACT(ctx))
opts = "+haf:Fp:co:O:sr:g:l:n:P:N:tb:B:ksiT";
else
- opts = "+hae:f:Fp:cC:dDGo:O:s:r:vg:l:n:P:N:tb:R:B:ksSiTm:M:H:q";
+ opts = "+hae:f:FA:p:cC:dDGo:O:s:r:vg:l:n:P:N:tb:R:B:ksSiTm:M:H:q";
c = getopt_long (argc-1, argv+1, opts, long_options, &option_index);
if (c == -1)
break;
+
+ /*
+ * If the current instance is to record a guest, then save
+ * all the arguments for this instance.
+ */
+ if (c != 'B' && c != 'A' && ctx->instance->flags & BUFFER_FL_GUEST) {
+ add_arg(ctx->instance, c, opts, long_options, optarg);
+ continue;
+ }
+
switch (c) {
case 'h':
usage(argv);
@@ -4663,6 +5096,26 @@ static void parse_record_options(int argc,
add_trigger(event, optarg);
break;
+ case 'A': {
+ char *name = NULL;
+ int cid = -1, port = -1;
+
+ if (!IS_RECORD(ctx))
+ die("-A is only allowed for record operations");
+
+ name = parse_guest_name(optarg, &cid, &port);
+ if (!name || cid == -1)
+ die("guest %s not found", optarg);
+ if (port == -1)
+ port = TRACE_AGENT_DEFAULT_PORT;
+
+ ctx->instance = create_instance(name);
+ ctx->instance->flags |= BUFFER_FL_GUEST;
+ ctx->instance->cid = cid;
+ ctx->instance->port = port;
+ add_instance(ctx->instance, 0);
+ break;
+ }
case 'F':
test_set_event_pid();
filter_task = 1;
@@ -4733,6 +5186,8 @@ static void parse_record_options(int argc,
ctx->disable = 1;
break;
case 'o':
+ if (IS_AGENT(ctx))
+ die("-o incompatible with agent recording");
if (host)
die("-o incompatible with -N");
if (IS_START(ctx))
@@ -4794,6 +5249,8 @@ static void parse_record_options(int argc,
case 'N':
if (!IS_RECORD(ctx))
die("-N only available with record");
+ if (IS_AGENT(ctx))
+ die("-N incompatible with agent recording");
if (ctx->output)
die("-N incompatible with -o");
host = optarg;
@@ -4890,6 +5347,16 @@ static void parse_record_options(int argc,
}
}
+ /* If --date is specified, prepend it to all guest VM flags */
+ if (ctx->date) {
+ struct buffer_instance *instance;
+
+ for_all_instances(instance) {
+ if (instance->flags & BUFFER_FL_GUEST)
+ add_argv(instance, "--date", true);
+ }
+ }
+
if (!ctx->filtered && ctx->instance->filter_mod)
add_func(&ctx->instance->filter_funcs,
ctx->instance->filter_mod, "*");
@@ -4920,7 +5387,8 @@ static enum trace_type get_trace_cmd_type(enum trace_cmd cmd)
{CMD_stream, TRACE_TYPE_STREAM},
{CMD_extract, TRACE_TYPE_EXTRACT},
{CMD_profile, TRACE_TYPE_STREAM},
- {CMD_start, TRACE_TYPE_START}
+ {CMD_start, TRACE_TYPE_START},
+ {CMD_record_agent, TRACE_TYPE_RECORD}
};
for (int i = 0; i < ARRAY_SIZE(trace_type_per_command); i++) {
@@ -4952,6 +5420,8 @@ static void finalize_record_trace(struct common_record_context *ctx)
if (instance->flags & BUFFER_FL_KEEP)
write_tracing_on(instance,
instance->tracing_on_init_val);
+ if (instance->flags & BUFFER_FL_AGENT)
+ tracecmd_output_close(instance->network_handle);
}
if (host)
@@ -4986,7 +5456,6 @@ static void record_trace(int argc, char **argv,
/* Save the state of tracing_on before starting */
for_all_instances(instance) {
-
if (!ctx->manual && instance->flags & BUFFER_FL_PROFILE)
enable_profile(instance);
@@ -5003,14 +5472,15 @@ static void record_trace(int argc, char **argv,
page_size = getpagesize();
- fset = set_ftrace(!ctx->disable, ctx->total_disable);
+ if (!(ctx->instance->flags & BUFFER_FL_GUEST))
+ fset = set_ftrace(!ctx->disable, ctx->total_disable);
tracecmd_disable_all_tracing(1);
for_all_instances(instance)
set_clock(instance);
/* Record records the date first */
- if (IS_RECORD(ctx) && ctx->date)
+ if ((IS_RECORD(ctx) || IS_AGENT(ctx)) && ctx->date)
ctx->date2ts = get_date_to_ts();
for_all_instances(instance) {
@@ -5045,9 +5515,13 @@ static void record_trace(int argc, char **argv,
exit(0);
}
- if (ctx->run_command)
+ if (ctx->run_command) {
run_cmd(type, (argc - optind) - 1, &argv[optind + 1]);
- else {
+ } else if (ctx->instance && (ctx->instance->flags & BUFFER_FL_AGENT)) {
+ update_task_filter();
+ tracecmd_enable_tracing();
+ tracecmd_msg_wait_close(ctx->instance->msg_handle);
+ } else {
update_task_filter();
tracecmd_enable_tracing();
/* We don't ptrace ourself */
@@ -5068,11 +5542,13 @@ static void record_trace(int argc, char **argv,
if (!keep)
tracecmd_disable_all_tracing(0);
- if (IS_RECORD(ctx)) {
- record_data(ctx);
- delete_thread_data();
- } else
- print_stats();
+ if (!IS_AGENT(ctx)) {
+ if (IS_RECORD(ctx)) {
+ record_data(ctx);
+ delete_thread_data();
+ } else
+ print_stats();
+ }
destroy_stats();
finalize_record_trace(ctx);
@@ -5202,3 +5678,40 @@ void trace_record(int argc, char **argv)
record_trace(argc, argv, &ctx);
exit(0);
}
+
+int trace_record_agent(struct tracecmd_msg_handle *msg_handle,
+ int cpus, int *fds,
+ int argc, char **argv)
+{
+ struct common_record_context ctx;
+ char **argv_plus;
+
+ /* Reset optind for getopt_long */
+ optind = 1;
+ /*
+ * argc is the number of elements in argv, but we need to convert
+ * argc and argv into "trace-cmd", "record", argv.
+ * where argc needs to grow by two.
+ */
+ argv_plus = calloc(argc + 2, sizeof(char *));
+ if (!argv_plus)
+ return -ENOMEM;
+
+ argv_plus[0] = "trace-cmd";
+ argv_plus[1] = "record";
+ memcpy(argv_plus + 2, argv, argc * sizeof(char *));
+ argc += 2;
+
+ parse_record_options(argc, argv_plus, CMD_record_agent, &ctx);
+ if (ctx.run_command)
+ return -EINVAL;
+
+ ctx.instance->fds = fds;
+ ctx.instance->flags |= BUFFER_FL_AGENT;
+ ctx.instance->msg_handle = msg_handle;
+ msg_handle->version = V3_PROTOCOL;
+ record_trace(argc, argv, &ctx);
+
+ free(argv_plus);
+ return 0;
+}
diff --git a/tracecmd/trace-usage.c b/tracecmd/trace-usage.c
index 9ea1906..93e85ba 100644
--- a/tracecmd/trace-usage.c
+++ b/tracecmd/trace-usage.c
@@ -231,11 +231,22 @@ static struct usage_help usage_help[] = {
"listen on a network socket for trace clients",
" %s listen -p port[-D][-o file][-d dir][-l logfile]\n"
" Creates a socket to listen for clients.\n"
- " -D create it in daemon mode.\n"
+ " -p port number to listen on.\n"
+ " -D run in daemon mode.\n"
" -o file name to use for clients.\n"
" -d directory to store client files.\n"
" -l logfile to write messages to.\n"
},
+#ifdef VSOCK
+ {
+ "agent",
+ "listen on a vsock socket for trace clients",
+ " %s agent -p port[-D]\n"
+ " Creates a vsock socket to listen for clients.\n"
+ " -p port number to listen on.\n"
+ " -D run in daemon mode.\n"
+ },
+#endif
{
"list",
"list the available events, plugins or options",
--
2.19.1
^ permalink raw reply related [flat|nested] 12+ messages in thread
* Re: [PATCH v3 3/6] trace-cmd: Add tracecmd_create_recorder_virt function
2019-01-14 15:27 ` [PATCH v3 3/6] trace-cmd: Add tracecmd_create_recorder_virt function Slavomir Kaslev
@ 2019-01-14 22:10 ` Steven Rostedt
2019-01-15 14:21 ` Slavomir Kaslev
0 siblings, 1 reply; 12+ messages in thread
From: Steven Rostedt @ 2019-01-14 22:10 UTC (permalink / raw)
To: Slavomir Kaslev; +Cc: linux-trace-devel, ykaradzhov, tstoyanov
On Mon, 14 Jan 2019 17:27:57 +0200
Slavomir Kaslev <kaslevs@vmware.com> wrote:
> @@ -194,8 +196,9 @@ tracecmd_create_buffer_recorder_fd(int fd, int cpu, unsigned flags, const char *
> return tracecmd_create_buffer_recorder_fd2(fd, -1, cpu, flags, buffer, 0);
> }
>
> -struct tracecmd_recorder *
> -tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags, const char *buffer)
> +static struct tracecmd_recorder *
> +__tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags,
> + const char *buffer)
> {
> struct tracecmd_recorder *recorder;
> int fd;
> @@ -258,6 +261,26 @@ tracecmd_create_buffer_recorder_maxkb(const char *file, int cpu, unsigned flags,
> goto out;
> }
>
> +struct tracecmd_recorder *
> +tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags,
> + const char *buffer)
> +{
> + return __tracecmd_create_buffer_recorder(file, cpu, flags, buffer);
> +}
> +
> +struct tracecmd_recorder *
> +tracecmd_create_recorder_virt(const char *file, int cpu, int trace_fd)
As this looks to be something that may be a library call someday, I
would keep flags as a parameter.
> +{
> + struct tracecmd_recorder *recorder;
> +
> + recorder = __tracecmd_create_buffer_recorder(
> + file, cpu, TRACECMD_RECORD_NOSPLICE, NULL);
Then you could pass this as:
recorder = __tracecmd_create_buffer_recorder(file, cpu,
flags | TRACECMD_RECORD_NOSPLICE, NULL);
-- Steve
> + if (recorder)
> + recorder->trace_fd = trace_fd;
> +
> + return recorder;
> +}
> +
> struct tracecmd_recorder *tracecmd_create_recorder_fd(int fd, int cpu, unsigned flags)
> {
> const char *tracing;
^ permalink raw reply [flat|nested] 12+ messages in thread
* Re: [PATCH v3 6/6] trace-cmd: Add VM kernel tracing over vsock sockets transport
2019-01-14 15:28 ` [PATCH v3 6/6] trace-cmd: Add VM kernel tracing over vsock sockets transport Slavomir Kaslev
@ 2019-01-14 22:46 ` Steven Rostedt
2019-01-15 15:00 ` Slavomir Kaslev
0 siblings, 1 reply; 12+ messages in thread
From: Steven Rostedt @ 2019-01-14 22:46 UTC (permalink / raw)
To: Slavomir Kaslev; +Cc: linux-trace-devel, ykaradzhov, tstoyanov
On Mon, 14 Jan 2019 17:28:00 +0200
Slavomir Kaslev <kaslevs@vmware.com> wrote:
\
> --- /dev/null
> +++ b/tracecmd/trace-agent.c
> @@ -0,0 +1,229 @@
> +// SPDX-License-Identifier: LGPL-2.1
> +/*
> + * Copyright (C) 2018 VMware Inc, Slavomir Kaslev <kaslevs@vmware.com>
> + *
> + * based on prior implementation by Yoshihiro Yunomae
> + * Copyright (C) 2013 Hitachi, Ltd.
> + * Yoshihiro YUNOMAE <yoshihiro.yunomae.ez@hitachi.com>
> + */
> +
> +#include <errno.h>
> +#include <fcntl.h>
> +#include <getopt.h>
> +#include <signal.h>
> +#include <stdbool.h>
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <sys/ioctl.h>
> +#include <sys/socket.h>
> +#include <sys/wait.h>
> +#include <unistd.h>
> +#include <linux/vm_sockets.h>
> +
> +#include "trace-local.h"
> +#include "trace-msg.h"
> +
> +static int make_vsock(unsigned port)
> +{
> + struct sockaddr_vm addr = {
> + .svm_family = AF_VSOCK,
> + .svm_cid = VMADDR_CID_ANY,
> + .svm_port = port,
> + };
> + int sd;
> +
> + sd = socket(AF_VSOCK, SOCK_STREAM, 0);
> + if (sd < 0)
> + return -1;
> +
> + setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &(int){1}, sizeof(int));
> +
> + if (bind(sd, (struct sockaddr *)&addr, sizeof(addr)))
> + return -1;
> +
> + if (listen(sd, SOMAXCONN))
> + return -1;
> +
> + return sd;
> +}
> +
> +static int get_vsock_port(int sd)
> +{
> + struct sockaddr_vm addr;
> + socklen_t addr_len = sizeof(addr);
> +
> + if (getsockname(sd, (struct sockaddr *)&addr, &addr_len))
> + return -1;
> +
> + if (addr.svm_family != AF_VSOCK)
> + return -1;
> +
> + return addr.svm_port;
> +}
> +
> +static void make_vsocks(int nr, int **fds, int **ports)
> +{
> + int i, fd, port;
> +
> + *fds = calloc(nr, sizeof(*fds));
> + *ports = calloc(nr, sizeof(*ports));
> + if (!*fds || !*ports)
> + die("Failed to allocate memory");
> +
> + for (i = 0; i < nr; i++) {
> + fd = make_vsock(VMADDR_PORT_ANY);
> + if (fd < 0)
> + die("Failed to open vsock socket");
> +
> + port = get_vsock_port(fd);
> + if (port < 0)
> + die("Failed to get vsock socket address");
> +
> + (*fds)[i] = fd;
> + (*ports)[i] = port;
> + }
> +}
> +
> +static void free_vsocks(int nr, int *fds, int *ports)
> +{
> + int i;
> +
> + for (i = 0; i < nr; i++)
> + close(fds[i]);
> + free(fds);
> + free(ports);
> +}
> +
> +static void agent_handle(int sd, int nr_cpus, int page_size)
> +{
> + struct tracecmd_msg_handle *msg_handle;
> + int *fds, *ports;
> + char **argv = NULL;
> + int argc = 0;
> +
> + msg_handle = tracecmd_msg_handle_alloc(sd, TRACECMD_MSG_FL_CLIENT);
> + if (!msg_handle)
> + die("Failed to allocate message handle");
> +
> + if (tracecmd_msg_recv_trace_req(msg_handle, &argc, &argv))
> + die("Failed to receive trace request");
> +
> + make_vsocks(nr_cpus, &fds, &ports);
> +
> + if (tracecmd_msg_send_trace_resp(msg_handle, nr_cpus, page_size, ports))
> + die("Failed to send trace response");
> +
> + trace_record_agent(msg_handle, nr_cpus, fds, argc, argv);
> +
> + free_vsocks(nr_cpus, fds, ports);
> + free(argv[0]);
> + free(argv);
> + tracecmd_msg_handle_close(msg_handle);
> + exit(0);
> +}
> +
> +static volatile pid_t handler_pid;
> +
> +static void handle_sigchld(int sig)
> +{
> + int wstatus;
> + pid_t pid;
> +
> + for (;;) {
> + pid = waitpid(-1, &wstatus, WNOHANG);
> + if (pid <= 0)
> + break;
> +
> + if (pid == handler_pid)
> + handler_pid = 0;
> + }
> +}
> +
> +static void agent_serve(unsigned port)
> +{
> + int sd, cd, nr_cpus;
> + pid_t pid;
> +
> + signal(SIGCHLD, handle_sigchld);
> +
> + nr_cpus = count_cpus();
> + page_size = getpagesize();
> +
> + sd = make_vsock(port);
> + if (sd < 0)
> + die("Failed to open vsock socket");
> +
> + for (;;) {
> + cd = accept(sd, NULL, NULL);
> + if (cd < 0) {
> + if (errno == EINTR)
> + continue;
> + die("accept");
> + }
> +
> + if (handler_pid)
> + goto busy;
> +
> + pid = fork();
> + if (pid == 0) {
> + close(sd);
> + signal(SIGCHLD, SIG_DFL);
> + agent_handle(cd, nr_cpus, page_size);
> + }
> + if (pid > 0)
> + handler_pid = pid;
> +
> + busy:
> + close(cd);
> + }
> +
Hmm, I don't see any break from the above for (;;) and that's an
infinite loop. That makes this dead code below.
> + close(sd);
> + signal(SIGCHLD, SIG_DFL);
> +}
> +
> +void trace_agent(int argc, char **argv)
> +{
> + bool do_daemon = false;
> + unsigned port = TRACE_AGENT_DEFAULT_PORT;
> +
> + if (argc < 2)
> + usage(argv);
> +
> + if (strcmp(argv[1], "agent") != 0)
> + usage(argv);
> +
> + for (;;) {
> + int c, option_index = 0;
> + static struct option long_options[] = {
> + {"port", required_argument, NULL, 'p'},
> + {"help", no_argument, NULL, '?'},
> + {NULL, 0, NULL, 0}
> + };
> +
> + c = getopt_long(argc-1, argv+1, "+hp:D",
> + long_options, &option_index);
> + if (c == -1)
> + break;
> + switch (c) {
> + case 'h':
> + usage(argv);
> + break;
> + case 'p':
> + port = atoi(optarg);
> + break;
> + case 'D':
> + do_daemon = true;
> + break;
> + default:
> + usage(argv);
> + }
> + }
> +
> + if ((argc - optind) >= 2)
> + usage(argv);
> +
> + if (do_daemon && daemon(1, 0))
> + die("daemon");
> +
> + agent_serve(port);
> +}
> diff --git a/tracecmd/trace-cmd.c b/tracecmd/trace-cmd.c
> index 797b303..3ae5e2e 100644
> --- a/tracecmd/trace-cmd.c
> +++ b/tracecmd/trace-cmd.c
> @@ -83,6 +83,9 @@ struct command commands[] = {
> {"hist", trace_hist},
> {"mem", trace_mem},
> {"listen", trace_listen},
> +#ifdef VSOCK
> + {"agent", trace_agent},
> +#endif
> {"split", trace_split},
> {"restore", trace_restore},
> {"stack", trace_stack},
> diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
> index 012371c..6c8754e 100644
> --- a/tracecmd/trace-record.c
> +++ b/tracecmd/trace-record.c
> @@ -33,6 +33,9 @@
> #include <errno.h>
> #include <limits.h>
> #include <libgen.h>
> +#ifdef VSOCK
> +#include <linux/vm_sockets.h>
> +#endif
>
> #include "trace-local.h"
> #include "trace-msg.h"
> @@ -74,8 +77,6 @@ static int buffers;
> static int clear_function_filters;
>
> static char *host;
> -static int *client_ports;
> -static int sfd;
>
> /* Max size to let a per cpu file get */
> static int max_kb;
> @@ -171,6 +172,15 @@ static struct tracecmd_recorder *recorder;
>
> static int ignore_event_not_found = 0;
>
> +static inline size_t grow_cap(size_t old_cap)
> +{
> + size_t cap = 3 * old_cap / 2;
> +
> + if (cap < 16)
> + cap = 16;
> + return cap;
> +}
I'm a bit confused by what the grow_cap() is for.
> +
> static inline int is_top_instance(struct buffer_instance *instance)
> {
> return instance == &top_instance;
> @@ -518,6 +528,36 @@ static char *get_temp_file(struct buffer_instance *instance, int cpu)
> return file;
> }
>
> +static char *get_guest_file(const char *file, const char *guest)
> +{
> + size_t guest_len = strlen(guest);
> + size_t file_len = strlen(file);
> + size_t base_len, idx = 0;
> + const char *p;
> + char *out;
> +
> + out = malloc(file_len + guest_len + 2 /* dash and \0 */);
> + if (!out)
> + return NULL;
> +
> + p = strrchr(file, '.');
> + if (p && p != file)
> + base_len = p - file;
> + else
> + base_len = file_len;
> +
> + memcpy(out, file, base_len);
> + idx += base_len;
> + out[idx++] = '-';
> + memcpy(out + idx, guest, guest_len);
> + idx += guest_len;
> + memcpy(out + idx, p, file_len - base_len);
> + idx += file_len - base_len;
> + out[idx] = '\0';
Can't we pretty much replace this whole function with:
p = strrchr(file, '.');
if (p && p != file) {
base_len = p - file;
else
base_len = strlen(file);
asprintf(&out, "%.*s-%s%s", base_len, file, guest, file + base_len);
?
> +
> + return out;
> +}
> +
> static void put_temp_file(char *file)
> {
> free(file);
> @@ -623,6 +663,16 @@ static void delete_thread_data(void)
> }
> }
>
> +static void tell_guests_to_stop(void)
> +{
> + struct buffer_instance *instance;
> +
> + for_all_instances(instance) {
> + if (instance->flags & BUFFER_FL_GUEST)
> + tracecmd_msg_handle_close(instance->msg_handle);
> + }
> +}
> +
> static void stop_threads(enum trace_type type)
> {
> struct timeval tv = { 0, 0 };
> @@ -632,6 +682,8 @@ static void stop_threads(enum trace_type type)
> if (!recorder_threads)
> return;
>
> + tell_guests_to_stop();
> +
> /* Tell all threads to finish up */
> for (i = 0; i < recorder_threads; i++) {
> if (pids[i].pid > 0) {
> @@ -2571,14 +2623,14 @@ static void flush(int sig)
> tracecmd_stop_recording(recorder);
> }
>
> -static void connect_port(int cpu)
> +static int connect_port(const char *host, unsigned port)
> {
> struct addrinfo hints;
> struct addrinfo *results, *rp;
> - int s;
> + int s, sfd;
> char buf[BUFSIZ];
>
> - snprintf(buf, BUFSIZ, "%d", client_ports[cpu]);
> + snprintf(buf, BUFSIZ, "%d", port);
>
> memset(&hints, 0, sizeof(hints));
> hints.ai_family = AF_UNSPEC;
> @@ -2605,7 +2657,190 @@ static void connect_port(int cpu)
>
> freeaddrinfo(results);
>
> - client_ports[cpu] = sfd;
> + return sfd;
> +}
> +
> +#ifdef VSOCK
> +static int open_vsock(unsigned cid, unsigned port)
> +{
> + struct sockaddr_vm addr = {
> + .svm_family = AF_VSOCK,
> + .svm_cid = cid,
> + .svm_port = port,
> + };
> + int sd;
> +
> + sd = socket(AF_VSOCK, SOCK_STREAM, 0);
> + if (sd < 0)
> + return -1;
> +
> + if (connect(sd, (struct sockaddr *)&addr, sizeof(addr)))
> + return -1;
> +
> + return sd;
> +}
> +#else
> +static inline int open_vsock(unsigned cid, unsigned port)
> +{
> + die("vsock is not supported");
> + return -1;
> +}
> +#endif
> +
> +static int do_accept(int sd)
> +{
> + int cd;
> +
> + for (;;) {
> + cd = accept(sd, NULL, NULL);
> + if (cd < 0) {
> + if (errno == EINTR)
> + continue;
> + die("accept");
> + }
> +
> + return cd;
> + }
> +
> + return -1;
> +}
> +
> +static bool is_digits(const char *s)
> +{
> + const char *p;
> +
> + for (p = s; *p; p++)
> + if (!isdigit(*p))
> + return false;
> +
> + return true;
> +}
> +
> +struct guest {
> + char *name;
> + int cid;
> + int pid;
> +};
> +
> +static size_t guests_cap, guests_len;
> +static struct guest *guests;
> +
> +static char *get_qemu_guest_name(char *arg)
> +{
> + char *tok, *end = arg;
> +
> + while ((tok = strsep(&end, ","))) {
> + if (strncmp(tok, "guest=", 6) == 0)
> + return tok + 6;
> + }
> +
> + return arg;
> +}
> +
> +static void read_qemu_guests(void)
> +{
> + static bool initialized = false;
> + struct dirent *entry;
> + char path[PATH_MAX];
> + DIR *dir;
> +
> + if (initialized)
> + return;
> +
> + initialized = true;
> + dir = opendir("/proc");
> + if (!dir)
> + die("opendir");
> +
> + for (entry = readdir(dir); entry; entry = readdir(dir)) {
> + bool is_qemu = false, last_was_name = false;
> + struct guest guest = {};
> + char *p, *arg = NULL;
> + size_t arg_size = 0;
> + FILE *f;
> +
> + if (!(entry->d_type == DT_DIR && is_digits(entry->d_name)))
> + continue;
> +
> + guest.pid = atoi(entry->d_name);
> + snprintf(path, sizeof(path), "/proc/%s/cmdline", entry->d_name);
> + f = fopen(path, "r");
> + if (!f)
> + continue;
> +
> + while (getdelim(&arg, &arg_size, 0, f) != -1) {
> + if (!is_qemu && strstr(arg, "qemu-system-")) {
> + is_qemu = true;
> + continue;
> + }
> +
> + if (!is_qemu)
> + continue;
> +
> + if (strcmp(arg, "-name") == 0) {
> + last_was_name = true;
> + continue;
> + }
> +
> + if (last_was_name) {
> + guest.name = strdup(get_qemu_guest_name(arg));
> + last_was_name = false;
> + continue;
> + }
> +
> + p = strstr(arg, "guest-cid=");
> + if (p) {
> + guest.cid = atoi(p + 10);
> + continue;
> + }
> + }
> +
> + if (is_qemu) {
> + if (guests_cap == guests_len) {
> + guests_cap = grow_cap(guests_cap);
This isn't a fast path is it? I'm not sure why we are using
guests_cap() here. Note, realloc allocs more than required and is
pretty much a nop if the allocated amount is still within its
allocation that it made the first time.
-- Steve
> + guests = realloc(guests,
> + guests_cap * sizeof(*guests));
> + }
> + guests[guests_len++] = guest;
> + }
> +
> + free(arg);
> + fclose(f);
> + }
> +
> + closedir(dir);
> +}
> +
>
^ permalink raw reply [flat|nested] 12+ messages in thread
* Re: [PATCH v3 3/6] trace-cmd: Add tracecmd_create_recorder_virt function
2019-01-14 22:10 ` Steven Rostedt
@ 2019-01-15 14:21 ` Slavomir Kaslev
2019-01-15 14:46 ` Steven Rostedt
0 siblings, 1 reply; 12+ messages in thread
From: Slavomir Kaslev @ 2019-01-15 14:21 UTC (permalink / raw)
To: Steven Rostedt; +Cc: linux-trace-devel, Yordan Karadzhov, Tzvetomir Stoyanov
On Tue, Jan 15, 2019 at 12:12 AM Steven Rostedt <rostedt@goodmis.org> wrote:
>
> On Mon, 14 Jan 2019 17:27:57 +0200
> Slavomir Kaslev <kaslevs@vmware.com> wrote:
>
>
> > @@ -194,8 +196,9 @@ tracecmd_create_buffer_recorder_fd(int fd, int cpu, unsigned flags, const char *
> > return tracecmd_create_buffer_recorder_fd2(fd, -1, cpu, flags, buffer, 0);
> > }
> >
> > -struct tracecmd_recorder *
> > -tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags, const char *buffer)
> > +static struct tracecmd_recorder *
> > +__tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags,
> > + const char *buffer)
> > {
> > struct tracecmd_recorder *recorder;
> > int fd;
> > @@ -258,6 +261,26 @@ tracecmd_create_buffer_recorder_maxkb(const char *file, int cpu, unsigned flags,
> > goto out;
> > }
> >
> > +struct tracecmd_recorder *
> > +tracecmd_create_buffer_recorder(const char *file, int cpu, unsigned flags,
> > + const char *buffer)
> > +{
> > + return __tracecmd_create_buffer_recorder(file, cpu, flags, buffer);
> > +}
> > +
> > +struct tracecmd_recorder *
> > +tracecmd_create_recorder_virt(const char *file, int cpu, int trace_fd)
>
> As this looks to be something that may be a library call someday, I
> would keep flags as a parameter.
>
> > +{
> > + struct tracecmd_recorder *recorder;
> > +
> > + recorder = __tracecmd_create_buffer_recorder(
> > + file, cpu, TRACECMD_RECORD_NOSPLICE, NULL);
>
> Then you could pass this as:
>
> recorder = __tracecmd_create_buffer_recorder(file, cpu,
> flags | TRACECMD_RECORD_NOSPLICE, NULL);
>
Makes sense but (continues below)
> -- Steve
>
> > + if (recorder)
> > + recorder->trace_fd = trace_fd;
it's this part that cannot be done outside of trace-recorder.c since
`recorder->trace_fd` is not exposed.
`tracecmd_create_recorder_virt` just allows a user to provide his own
file descriptor from where tracing data should be read (in this case
from the agent's vsock sockets)
On an unrelated note, the TRACECMD_RECORD_NOSPLICE flag is only needed
on Linux version <4.20. Do we want to detect if splicing on vsock
sockets works on startup and use this flag only if necessary?
> > +
> > + return recorder;
> > +}
> > +
> > struct tracecmd_recorder *tracecmd_create_recorder_fd(int fd, int cpu, unsigned flags)
> > {
> > const char *tracing;
>
^ permalink raw reply [flat|nested] 12+ messages in thread
* Re: [PATCH v3 3/6] trace-cmd: Add tracecmd_create_recorder_virt function
2019-01-15 14:21 ` Slavomir Kaslev
@ 2019-01-15 14:46 ` Steven Rostedt
0 siblings, 0 replies; 12+ messages in thread
From: Steven Rostedt @ 2019-01-15 14:46 UTC (permalink / raw)
To: Slavomir Kaslev; +Cc: linux-trace-devel, Yordan Karadzhov, Tzvetomir Stoyanov
On Tue, 15 Jan 2019 14:21:11 +0000
Slavomir Kaslev <kaslevs@vmware.com> wrote:
> On an unrelated note, the TRACECMD_RECORD_NOSPLICE flag is only needed
> on Linux version <4.20. Do we want to detect if splicing on vsock
> sockets works on startup and use this flag only if necessary?
Yes, sounds reasonable.
-- Steve
^ permalink raw reply [flat|nested] 12+ messages in thread
* Re: [PATCH v3 6/6] trace-cmd: Add VM kernel tracing over vsock sockets transport
2019-01-14 22:46 ` Steven Rostedt
@ 2019-01-15 15:00 ` Slavomir Kaslev
0 siblings, 0 replies; 12+ messages in thread
From: Slavomir Kaslev @ 2019-01-15 15:00 UTC (permalink / raw)
To: Steven Rostedt; +Cc: linux-trace-devel, Yordan Karadzhov, Tzvetomir Stoyanov
On Tue, Jan 15, 2019 at 12:46 AM Steven Rostedt <rostedt@goodmis.org> wrote:
>
> On Mon, 14 Jan 2019 17:28:00 +0200
> Slavomir Kaslev <kaslevs@vmware.com> wrote:
> \
> > --- /dev/null
> > +++ b/tracecmd/trace-agent.c
> > @@ -0,0 +1,229 @@
> > +// SPDX-License-Identifier: LGPL-2.1
> > +/*
> > + * Copyright (C) 2018 VMware Inc, Slavomir Kaslev <kaslevs@vmware.com>
> > + *
> > + * based on prior implementation by Yoshihiro Yunomae
> > + * Copyright (C) 2013 Hitachi, Ltd.
> > + * Yoshihiro YUNOMAE <yoshihiro.yunomae.ez@hitachi.com>
> > + */
> > +
> > +#include <errno.h>
> > +#include <fcntl.h>
> > +#include <getopt.h>
> > +#include <signal.h>
> > +#include <stdbool.h>
> > +#include <stdio.h>
> > +#include <stdlib.h>
> > +#include <sys/ioctl.h>
> > +#include <sys/socket.h>
> > +#include <sys/wait.h>
> > +#include <unistd.h>
> > +#include <linux/vm_sockets.h>
> > +
> > +#include "trace-local.h"
> > +#include "trace-msg.h"
> > +
> > +static int make_vsock(unsigned port)
> > +{
> > + struct sockaddr_vm addr = {
> > + .svm_family = AF_VSOCK,
> > + .svm_cid = VMADDR_CID_ANY,
> > + .svm_port = port,
> > + };
> > + int sd;
> > +
> > + sd = socket(AF_VSOCK, SOCK_STREAM, 0);
> > + if (sd < 0)
> > + return -1;
> > +
> > + setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &(int){1}, sizeof(int));
> > +
> > + if (bind(sd, (struct sockaddr *)&addr, sizeof(addr)))
> > + return -1;
> > +
> > + if (listen(sd, SOMAXCONN))
> > + return -1;
> > +
> > + return sd;
> > +}
> > +
> > +static int get_vsock_port(int sd)
> > +{
> > + struct sockaddr_vm addr;
> > + socklen_t addr_len = sizeof(addr);
> > +
> > + if (getsockname(sd, (struct sockaddr *)&addr, &addr_len))
> > + return -1;
> > +
> > + if (addr.svm_family != AF_VSOCK)
> > + return -1;
> > +
> > + return addr.svm_port;
> > +}
> > +
> > +static void make_vsocks(int nr, int **fds, int **ports)
> > +{
> > + int i, fd, port;
> > +
> > + *fds = calloc(nr, sizeof(*fds));
> > + *ports = calloc(nr, sizeof(*ports));
> > + if (!*fds || !*ports)
> > + die("Failed to allocate memory");
> > +
> > + for (i = 0; i < nr; i++) {
> > + fd = make_vsock(VMADDR_PORT_ANY);
> > + if (fd < 0)
> > + die("Failed to open vsock socket");
> > +
> > + port = get_vsock_port(fd);
> > + if (port < 0)
> > + die("Failed to get vsock socket address");
> > +
> > + (*fds)[i] = fd;
> > + (*ports)[i] = port;
> > + }
> > +}
> > +
> > +static void free_vsocks(int nr, int *fds, int *ports)
> > +{
> > + int i;
> > +
> > + for (i = 0; i < nr; i++)
> > + close(fds[i]);
> > + free(fds);
> > + free(ports);
> > +}
> > +
> > +static void agent_handle(int sd, int nr_cpus, int page_size)
> > +{
> > + struct tracecmd_msg_handle *msg_handle;
> > + int *fds, *ports;
> > + char **argv = NULL;
> > + int argc = 0;
> > +
> > + msg_handle = tracecmd_msg_handle_alloc(sd, TRACECMD_MSG_FL_CLIENT);
> > + if (!msg_handle)
> > + die("Failed to allocate message handle");
> > +
> > + if (tracecmd_msg_recv_trace_req(msg_handle, &argc, &argv))
> > + die("Failed to receive trace request");
> > +
> > + make_vsocks(nr_cpus, &fds, &ports);
> > +
> > + if (tracecmd_msg_send_trace_resp(msg_handle, nr_cpus, page_size, ports))
> > + die("Failed to send trace response");
> > +
> > + trace_record_agent(msg_handle, nr_cpus, fds, argc, argv);
> > +
> > + free_vsocks(nr_cpus, fds, ports);
> > + free(argv[0]);
> > + free(argv);
> > + tracecmd_msg_handle_close(msg_handle);
> > + exit(0);
> > +}
> > +
> > +static volatile pid_t handler_pid;
> > +
> > +static void handle_sigchld(int sig)
> > +{
> > + int wstatus;
> > + pid_t pid;
> > +
> > + for (;;) {
> > + pid = waitpid(-1, &wstatus, WNOHANG);
> > + if (pid <= 0)
> > + break;
> > +
> > + if (pid == handler_pid)
> > + handler_pid = 0;
> > + }
> > +}
> > +
> > +static void agent_serve(unsigned port)
> > +{
> > + int sd, cd, nr_cpus;
> > + pid_t pid;
> > +
> > + signal(SIGCHLD, handle_sigchld);
> > +
> > + nr_cpus = count_cpus();
> > + page_size = getpagesize();
> > +
> > + sd = make_vsock(port);
> > + if (sd < 0)
> > + die("Failed to open vsock socket");
> > +
> > + for (;;) {
> > + cd = accept(sd, NULL, NULL);
> > + if (cd < 0) {
> > + if (errno == EINTR)
> > + continue;
> > + die("accept");
> > + }
> > +
> > + if (handler_pid)
> > + goto busy;
> > +
> > + pid = fork();
> > + if (pid == 0) {
> > + close(sd);
> > + signal(SIGCHLD, SIG_DFL);
> > + agent_handle(cd, nr_cpus, page_size);
> > + }
> > + if (pid > 0)
> > + handler_pid = pid;
> > +
> > + busy:
> > + close(cd);
> > + }
> > +
>
> Hmm, I don't see any break from the above for (;;) and that's an
> infinite loop. That makes this dead code below.
>
> > + close(sd);
> > + signal(SIGCHLD, SIG_DFL);
> > +}
>
>
>
> > +
> > +void trace_agent(int argc, char **argv)
> > +{
> > + bool do_daemon = false;
> > + unsigned port = TRACE_AGENT_DEFAULT_PORT;
> > +
> > + if (argc < 2)
> > + usage(argv);
> > +
> > + if (strcmp(argv[1], "agent") != 0)
> > + usage(argv);
> > +
> > + for (;;) {
> > + int c, option_index = 0;
> > + static struct option long_options[] = {
> > + {"port", required_argument, NULL, 'p'},
> > + {"help", no_argument, NULL, '?'},
> > + {NULL, 0, NULL, 0}
> > + };
> > +
> > + c = getopt_long(argc-1, argv+1, "+hp:D",
> > + long_options, &option_index);
> > + if (c == -1)
> > + break;
> > + switch (c) {
> > + case 'h':
> > + usage(argv);
> > + break;
> > + case 'p':
> > + port = atoi(optarg);
> > + break;
> > + case 'D':
> > + do_daemon = true;
> > + break;
> > + default:
> > + usage(argv);
> > + }
> > + }
> > +
> > + if ((argc - optind) >= 2)
> > + usage(argv);
> > +
> > + if (do_daemon && daemon(1, 0))
> > + die("daemon");
> > +
> > + agent_serve(port);
> > +}
> > diff --git a/tracecmd/trace-cmd.c b/tracecmd/trace-cmd.c
> > index 797b303..3ae5e2e 100644
> > --- a/tracecmd/trace-cmd.c
> > +++ b/tracecmd/trace-cmd.c
> > @@ -83,6 +83,9 @@ struct command commands[] = {
> > {"hist", trace_hist},
> > {"mem", trace_mem},
> > {"listen", trace_listen},
> > +#ifdef VSOCK
> > + {"agent", trace_agent},
> > +#endif
> > {"split", trace_split},
> > {"restore", trace_restore},
> > {"stack", trace_stack},
> > diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
> > index 012371c..6c8754e 100644
> > --- a/tracecmd/trace-record.c
> > +++ b/tracecmd/trace-record.c
> > @@ -33,6 +33,9 @@
> > #include <errno.h>
> > #include <limits.h>
> > #include <libgen.h>
> > +#ifdef VSOCK
> > +#include <linux/vm_sockets.h>
> > +#endif
> >
> > #include "trace-local.h"
> > #include "trace-msg.h"
> > @@ -74,8 +77,6 @@ static int buffers;
> > static int clear_function_filters;
> >
> > static char *host;
> > -static int *client_ports;
> > -static int sfd;
> >
> > /* Max size to let a per cpu file get */
> > static int max_kb;
> > @@ -171,6 +172,15 @@ static struct tracecmd_recorder *recorder;
> >
> > static int ignore_event_not_found = 0;
> >
> > +static inline size_t grow_cap(size_t old_cap)
> > +{
> > + size_t cap = 3 * old_cap / 2;
> > +
> > + if (cap < 16)
> > + cap = 16;
> > + return cap;
> > +}
>
> I'm a bit confused by what the grow_cap() is for.
I did some benchmarking awhile ago and calling realloc on each new
entry when appending is indeed linear (which implies that realloc
amortizes to constant).
grow_cap() grows the buffer exponentially and hence calls realloc only
log(n) times, which turns out to be again linear but with much smaller
constant.
Git uses something similar they call ALLOC_GROW:
https://github.com/git/git/blob/0f086e6dcabfbf059e7483ebd7d41080faec3d77/cache.h#L650
I agree that neither of the current uses of grow_cap are not on
performance critical paths so I'll drop it in the next version of this
series.
>
> > +
> > static inline int is_top_instance(struct buffer_instance *instance)
> > {
> > return instance == &top_instance;
> > @@ -518,6 +528,36 @@ static char *get_temp_file(struct buffer_instance *instance, int cpu)
> > return file;
> > }
> >
> > +static char *get_guest_file(const char *file, const char *guest)
> > +{
> > + size_t guest_len = strlen(guest);
> > + size_t file_len = strlen(file);
> > + size_t base_len, idx = 0;
> > + const char *p;
> > + char *out;
> > +
> > + out = malloc(file_len + guest_len + 2 /* dash and \0 */);
> > + if (!out)
> > + return NULL;
> > +
> > + p = strrchr(file, '.');
> > + if (p && p != file)
> > + base_len = p - file;
> > + else
> > + base_len = file_len;
> > +
> > + memcpy(out, file, base_len);
> > + idx += base_len;
> > + out[idx++] = '-';
> > + memcpy(out + idx, guest, guest_len);
> > + idx += guest_len;
> > + memcpy(out + idx, p, file_len - base_len);
> > + idx += file_len - base_len;
> > + out[idx] = '\0';
>
> Can't we pretty much replace this whole function with:
>
> p = strrchr(file, '.');
> if (p && p != file) {
> base_len = p - file;
> else
> base_len = strlen(file);
>
> asprintf(&out, "%.*s-%s%s", base_len, file, guest, file + base_len);
>
> ?
Good catch. This is a leftover from the virt-server branch which I
simplified but didn't know about the %.*s format magic.
>
>
> > +
> > + return out;
> > +}
> > +
> > static void put_temp_file(char *file)
> > {
> > free(file);
> > @@ -623,6 +663,16 @@ static void delete_thread_data(void)
> > }
> > }
> >
> > +static void tell_guests_to_stop(void)
> > +{
> > + struct buffer_instance *instance;
> > +
> > + for_all_instances(instance) {
> > + if (instance->flags & BUFFER_FL_GUEST)
> > + tracecmd_msg_handle_close(instance->msg_handle);
> > + }
> > +}
> > +
> > static void stop_threads(enum trace_type type)
> > {
> > struct timeval tv = { 0, 0 };
> > @@ -632,6 +682,8 @@ static void stop_threads(enum trace_type type)
> > if (!recorder_threads)
> > return;
> >
> > + tell_guests_to_stop();
> > +
> > /* Tell all threads to finish up */
> > for (i = 0; i < recorder_threads; i++) {
> > if (pids[i].pid > 0) {
> > @@ -2571,14 +2623,14 @@ static void flush(int sig)
> > tracecmd_stop_recording(recorder);
> > }
> >
> > -static void connect_port(int cpu)
> > +static int connect_port(const char *host, unsigned port)
> > {
> > struct addrinfo hints;
> > struct addrinfo *results, *rp;
> > - int s;
> > + int s, sfd;
> > char buf[BUFSIZ];
> >
> > - snprintf(buf, BUFSIZ, "%d", client_ports[cpu]);
> > + snprintf(buf, BUFSIZ, "%d", port);
> >
> > memset(&hints, 0, sizeof(hints));
> > hints.ai_family = AF_UNSPEC;
> > @@ -2605,7 +2657,190 @@ static void connect_port(int cpu)
> >
> > freeaddrinfo(results);
> >
> > - client_ports[cpu] = sfd;
> > + return sfd;
> > +}
> > +
> > +#ifdef VSOCK
> > +static int open_vsock(unsigned cid, unsigned port)
> > +{
> > + struct sockaddr_vm addr = {
> > + .svm_family = AF_VSOCK,
> > + .svm_cid = cid,
> > + .svm_port = port,
> > + };
> > + int sd;
> > +
> > + sd = socket(AF_VSOCK, SOCK_STREAM, 0);
> > + if (sd < 0)
> > + return -1;
> > +
> > + if (connect(sd, (struct sockaddr *)&addr, sizeof(addr)))
> > + return -1;
> > +
> > + return sd;
> > +}
> > +#else
> > +static inline int open_vsock(unsigned cid, unsigned port)
> > +{
> > + die("vsock is not supported");
> > + return -1;
> > +}
> > +#endif
> > +
> > +static int do_accept(int sd)
> > +{
> > + int cd;
> > +
> > + for (;;) {
> > + cd = accept(sd, NULL, NULL);
> > + if (cd < 0) {
> > + if (errno == EINTR)
> > + continue;
> > + die("accept");
> > + }
> > +
> > + return cd;
> > + }
> > +
> > + return -1;
> > +}
> > +
> > +static bool is_digits(const char *s)
> > +{
> > + const char *p;
> > +
> > + for (p = s; *p; p++)
> > + if (!isdigit(*p))
> > + return false;
> > +
> > + return true;
> > +}
> > +
> > +struct guest {
> > + char *name;
> > + int cid;
> > + int pid;
> > +};
> > +
> > +static size_t guests_cap, guests_len;
> > +static struct guest *guests;
> > +
> > +static char *get_qemu_guest_name(char *arg)
> > +{
> > + char *tok, *end = arg;
> > +
> > + while ((tok = strsep(&end, ","))) {
> > + if (strncmp(tok, "guest=", 6) == 0)
> > + return tok + 6;
> > + }
> > +
> > + return arg;
> > +}
> > +
> > +static void read_qemu_guests(void)
> > +{
> > + static bool initialized = false;
> > + struct dirent *entry;
> > + char path[PATH_MAX];
> > + DIR *dir;
> > +
> > + if (initialized)
> > + return;
> > +
> > + initialized = true;
> > + dir = opendir("/proc");
> > + if (!dir)
> > + die("opendir");
> > +
> > + for (entry = readdir(dir); entry; entry = readdir(dir)) {
> > + bool is_qemu = false, last_was_name = false;
> > + struct guest guest = {};
> > + char *p, *arg = NULL;
> > + size_t arg_size = 0;
> > + FILE *f;
> > +
> > + if (!(entry->d_type == DT_DIR && is_digits(entry->d_name)))
> > + continue;
> > +
> > + guest.pid = atoi(entry->d_name);
> > + snprintf(path, sizeof(path), "/proc/%s/cmdline", entry->d_name);
> > + f = fopen(path, "r");
> > + if (!f)
> > + continue;
> > +
> > + while (getdelim(&arg, &arg_size, 0, f) != -1) {
> > + if (!is_qemu && strstr(arg, "qemu-system-")) {
> > + is_qemu = true;
> > + continue;
> > + }
> > +
> > + if (!is_qemu)
> > + continue;
> > +
> > + if (strcmp(arg, "-name") == 0) {
> > + last_was_name = true;
> > + continue;
> > + }
> > +
> > + if (last_was_name) {
> > + guest.name = strdup(get_qemu_guest_name(arg));
> > + last_was_name = false;
> > + continue;
> > + }
> > +
> > + p = strstr(arg, "guest-cid=");
> > + if (p) {
> > + guest.cid = atoi(p + 10);
> > + continue;
> > + }
> > + }
> > +
> > + if (is_qemu) {
> > + if (guests_cap == guests_len) {
> > + guests_cap = grow_cap(guests_cap);
>
> This isn't a fast path is it? I'm not sure why we are using
> guests_cap() here. Note, realloc allocs more than required and is
> pretty much a nop if the allocated amount is still within its
> allocation that it made the first time.
>
> -- Steve
>
>
> > + guests = realloc(guests,
> > + guests_cap * sizeof(*guests));
> > + }
> > + guests[guests_len++] = guest;
> > + }
> > +
> > + free(arg);
> > + fclose(f);
> > + }
> > +
> > + closedir(dir);
> > +}
> > +
> >
^ permalink raw reply [flat|nested] 12+ messages in thread
end of thread, other threads:[~2019-01-15 15:00 UTC | newest]
Thread overview: 12+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-01-14 15:27 [PATCH v3 0/6] Add VM kernel tracing over vsock sockets Slavomir Kaslev
2019-01-14 15:27 ` [PATCH v3 1/6] trace-cmd: Minor refactoring Slavomir Kaslev
2019-01-14 15:27 ` [PATCH v3 2/6] trace-cmd: Detect if vsock sockets are available Slavomir Kaslev
2019-01-14 15:27 ` [PATCH v3 3/6] trace-cmd: Add tracecmd_create_recorder_virt function Slavomir Kaslev
2019-01-14 22:10 ` Steven Rostedt
2019-01-15 14:21 ` Slavomir Kaslev
2019-01-15 14:46 ` Steven Rostedt
2019-01-14 15:27 ` [PATCH v3 4/6] trace-cmd: Add TRACE_REQ and TRACE_RESP messages Slavomir Kaslev
2019-01-14 15:27 ` [PATCH v3 5/6] trace-cmd: Add buffer instance flags for tracing in guest and agent context Slavomir Kaslev
2019-01-14 15:28 ` [PATCH v3 6/6] trace-cmd: Add VM kernel tracing over vsock sockets transport Slavomir Kaslev
2019-01-14 22:46 ` Steven Rostedt
2019-01-15 15:00 ` Slavomir Kaslev
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).