[PATCH 00/26] trace-cmd: Add agent proxy (agent on the host)

[PATCH 00/26] trace-cmd: Add agent proxy (agent on the host)

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

Currently the trace-cmd agent is expected to run on the guest with the host
running the recorder. There is a lot of assumptions throughout the code that
expects this. Rearrange the code and allow the guest to be the recorder and
the host act as an agent via the -P (proxy) option. The guest record can
call the agent proxy on the host with the --proxy option.

Currently this does not handle the host being a proxy for other guests, but
that will come later. For now, this code will allow the host to run the
agent and handle the time synchronizations needed between the two (and pass
the data back up to the guest). The siblings will be another project, but
for now we at least have the host agent working with the guest and having
the timings all in sync.

Joel Fernandes (1):
  trace-cmd: Move trace_msg cache file to memfd

Steven Rostedt (Google) (25):
  trace-cmd record: Make start_threads() static
  trace-cmd: Move add_guest_info() into trace-vm.c
  trace-cmd: Simplify add_guest()
  trace-cmd: Move find_pid_by_cid() into add_guest()
  trace-cmd: Move find_tasks() into add_guest()
  trace-cmd Makefile: Change test-build to link as well
  trace-cmd agent: Test if memfd_create() is available
  trace-cmd: Add kernel-doc to trace_record_agent()
  trace-cmd: Move selecting tsync protocol out of tracecmd_tsync_with_host()
  trace-cmd: Move accepting tsync connection out of tracecmd_tsync_with_host()
  trace-cmd: Have get_vsocket_params() cid and rcid parameters be optional
  trace-cmd agent: Add trace_tsync_as_guest() helper function
  trace-cmd record: Pass cpu_count instead of an instance to stop_mapping_vcpus()
  trace-cmd record: Add trace_tsync_as_host() helper
  trace-cmd: Move tsync as guest and host helpers into trace-tsync.c
  trace-cmd msg: Add PROXY communication
  trace-cmd: Add agent proxy communications between record and agent
  trace-cmd msg: Keep track of offset of flushed cache
  trace-cmd library: Add tracecmd_prepare_options()
  trace-cmd library: Add tracecmd_msg_flush_data()
  trace-cmd agent proxy: Allow agent to send more meta data after trace
  trace-cmd agent proxy: Add the remote guest cid to guest list
  trace-cmd agent-proxy: Send options at the end of the trace
  trace-cmd: Have the guest structure hold guest trace_id
  trace-cmd: Have the host agent proxy control the time synchronization

 Makefile                                      |  13 +-
 .../include/private/trace-cmd-private.h       |  35 +-
 lib/trace-cmd/include/trace-cmd-local.h       |   2 +
 lib/trace-cmd/trace-msg.c                     | 292 ++++++++++--
 lib/trace-cmd/trace-output.c                  | 243 +++++++++-
 lib/trace-cmd/trace-timesync.c                |  72 +--
 tracecmd/Makefile                             |   1 +
 tracecmd/include/trace-local.h                |  24 +-
 tracecmd/trace-agent.c                        | 130 ++++-
 tracecmd/trace-record.c                       | 444 +++++++-----------
 tracecmd/trace-tsync.c                        | 244 ++++++++++
 tracecmd/trace-vm.c                           | 138 +++++-
 tracecmd/trace-vsock.c                        |  32 +-
 13 files changed, 1205 insertions(+), 465 deletions(-)
 create mode 100644 tracecmd/trace-tsync.c

-- 
2.35.1

[PATCH 01/26] trace-cmd record: Make start_threads() static

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

The function start_threads() is only used by trace-record.c, no reason
that it is not a static function.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/trace-record.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index 27c4e7ba6f3f..a22935263625 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -4187,7 +4187,7 @@ static void setup_agent(struct buffer_instance *instance,
 	instance->network_handle = network_handle;
 }
 
-void start_threads(enum trace_type type, struct common_record_context *ctx)
+static void start_threads(enum trace_type type, struct common_record_context *ctx)
 {
 	struct buffer_instance *instance;
 	int total_cpu_count = 0;
-- 
2.35.1

[PATCH 02/26] trace-cmd: Move add_guest_info() into trace-vm.c

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

In preparation for the agent proxy, move add_guest_info() into trace-vm.c
and rename it to trace_add_guest_info(). This will be used by the agent
when acting as a proxy.

Also comment the function to state what it does and what it records into
the trace.dat option.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/include/trace-local.h |  1 +
 tracecmd/trace-record.c        | 54 +-----------------------------
 tracecmd/trace-vm.c            | 61 ++++++++++++++++++++++++++++++++++
 3 files changed, 63 insertions(+), 53 deletions(-)

diff --git a/tracecmd/include/trace-local.h b/tracecmd/include/trace-local.h
index e3fec1319880..f3b805fa7bad 100644
--- a/tracecmd/include/trace-local.h
+++ b/tracecmd/include/trace-local.h
@@ -339,6 +339,7 @@ bool trace_have_guests_pid(void);
 void read_qemu_guests(void);
 int get_guest_pid(unsigned int guest_cid);
 int get_guest_vcpu_pid(unsigned int guest_cid, unsigned int guest_vcpu);
+void trace_add_guest_info(struct tracecmd_output *handle, struct buffer_instance *instance);
 
 /* moved from trace-cmd.h */
 void tracecmd_remove_instances(void);
diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index a22935263625..67ad1208f90c 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -4297,58 +4297,6 @@ static void append_buffer(struct tracecmd_output *handle,
 	}
 }
 
-static void
-add_guest_info(struct tracecmd_output *handle, struct buffer_instance *instance)
-{
-	struct trace_guest *guest;
-	const char *name;
-	char *buf, *p;
-	int size;
-	int pid;
-	int i;
-
-	if (is_network(instance)) {
-		name = instance->name;
-	} else {
-		guest = trace_get_guest(instance->cid, NULL);
-		if (!guest)
-			return;
-		name = guest->name;
-	}
-
-	size = strlen(name) + 1;
-	size += sizeof(long long);	/* trace_id */
-	size += sizeof(int);		/* cpu count */
-	size += instance->cpu_count * 2 * sizeof(int);	/* cpu,pid pair */
-
-	buf = calloc(1, size);
-	if (!buf)
-		return;
-	p = buf;
-	strcpy(p, name);
-	p += strlen(name) + 1;
-
-	memcpy(p, &instance->trace_id, sizeof(long long));
-	p += sizeof(long long);
-
-	memcpy(p, &instance->cpu_count, sizeof(int));
-	p += sizeof(int);
-	for (i = 0; i < instance->cpu_count; i++) {
-		pid = -1;
-		if (!is_network(instance)) {
-			if (i < guest->cpu_max)
-				pid = guest->cpu_pid[i];
-		}
-		memcpy(p, &i, sizeof(int));
-		p += sizeof(int);
-		memcpy(p, &pid, sizeof(int));
-		p += sizeof(int);
-	}
-
-	tracecmd_add_option(handle, TRACECMD_OPTION_GUEST, size, buf);
-	free(buf);
-}
-
 static void
 add_pid_maps(struct tracecmd_output *handle, struct buffer_instance *instance)
 {
@@ -4679,7 +4627,7 @@ static void record_data(struct common_record_context *ctx)
 
 		for_all_instances(instance) {
 			if (is_guest(instance))
-				add_guest_info(handle, instance);
+				trace_add_guest_info(handle, instance);
 		}
 
 		if (ctx->tsc2nsec.mult) {
diff --git a/tracecmd/trace-vm.c b/tracecmd/trace-vm.c
index 57dbef8d42e4..47f8d96a4015 100644
--- a/tracecmd/trace-vm.c
+++ b/tracecmd/trace-vm.c
@@ -386,3 +386,64 @@ int get_guest_vcpu_pid(unsigned int guest_cid, unsigned int guest_vcpu)
 	}
 	return -1;
 }
+
+/**
+ * trace_add_guest_info - Add the guest info into the trace file option
+ * @handle: The file handle that the guest info option is added to
+ * @instance: The instance that that represents the guest
+ *
+ * Adds information about the guest from the @instance into an option
+ * for the @instance. It records the trace_id, the number of CPUs,
+ * as well as the PIDs of the host that represent the CPUs.
+ */
+void
+trace_add_guest_info(struct tracecmd_output *handle, struct buffer_instance *instance)
+{
+	struct trace_guest *guest;
+	const char *name;
+	char *buf, *p;
+	int size;
+	int pid;
+	int i;
+
+	if (is_network(instance)) {
+		name = instance->name;
+	} else {
+		guest = trace_get_guest(instance->cid, NULL);
+		if (!guest)
+			return;
+		name = guest->name;
+	}
+
+	size = strlen(name) + 1;
+	size += sizeof(long long);	/* trace_id */
+	size += sizeof(int);		/* cpu count */
+	size += instance->cpu_count * 2 * sizeof(int);	/* cpu,pid pair */
+
+	buf = calloc(1, size);
+	if (!buf)
+		return;
+	p = buf;
+	strcpy(p, name);
+	p += strlen(name) + 1;
+
+	memcpy(p, &instance->trace_id, sizeof(long long));
+	p += sizeof(long long);
+
+	memcpy(p, &instance->cpu_count, sizeof(int));
+	p += sizeof(int);
+	for (i = 0; i < instance->cpu_count; i++) {
+		pid = -1;
+		if (!is_network(instance)) {
+			if (i < guest->cpu_max)
+				pid = guest->cpu_pid[i];
+		}
+		memcpy(p, &i, sizeof(int));
+		p += sizeof(int);
+		memcpy(p, &pid, sizeof(int));
+		p += sizeof(int);
+	}
+
+	tracecmd_add_option(handle, TRACECMD_OPTION_GUEST, size, buf);
+	free(buf);
+}
-- 
2.35.1

[PATCH 03/26] trace-cmd: Simplify add_guest()

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

Simplify the function add_guest by using a local variable to point to the
array element instead of dereferencing the array every time.

Link: https://lore.kernel.org/linux-trace-devel/20220428150635.789051-3-rostedt@goodmis.org

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/trace-vm.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/tracecmd/trace-vm.c b/tracecmd/trace-vm.c
index 47f8d96a4015..7a91038d75dc 100644
--- a/tracecmd/trace-vm.c
+++ b/tracecmd/trace-vm.c
@@ -56,18 +56,22 @@ bool trace_have_guests_pid(void)
 
 static struct trace_guest *add_guest(unsigned int cid, const char *name)
 {
+	struct trace_guest *guest;
+
 	guests = realloc(guests, (guests_len + 1) * sizeof(*guests));
 	if (!guests)
 		die("allocating new guest");
-	memset(&guests[guests_len], 0, sizeof(struct trace_guest));
-	guests[guests_len].name = strdup(name);
-	if (!guests[guests_len].name)
+
+	guest = &guests[guests_len++];
+
+	memset(guest, 0, sizeof(*guest));
+	guest->name = strdup(name);
+	if (!guest->name)
 		die("allocating guest name");
-	guests[guests_len].cid = cid;
-	guests[guests_len].pid = -1;
-	guests_len++;
+	guest->cid = cid;
+	guest->pid = -1;
 
-	return &guests[guests_len - 1];
+	return guest;
 }
 
 static struct tracefs_instance *start_trace_connect(void)
-- 
2.35.1

[PATCH 04/26] trace-cmd: Move find_pid_by_cid() into add_guest()

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

To simplify the code even more, move the find_pid_by_cid() into
add_guest() to map the main pid process to the cid.

Link: https://lore.kernel.org/linux-trace-devel/20220428150635.789051-5-rostedt@goodmis.org

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/trace-vm.c | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/tracecmd/trace-vm.c b/tracecmd/trace-vm.c
index 7a91038d75dc..4b4e039bd2cb 100644
--- a/tracecmd/trace-vm.c
+++ b/tracecmd/trace-vm.c
@@ -54,6 +54,8 @@ bool trace_have_guests_pid(void)
 	return true;
 }
 
+static void find_pid_by_cid(struct trace_guest *guest);
+
 static struct trace_guest *add_guest(unsigned int cid, const char *name)
 {
 	struct trace_guest *guest;
@@ -71,6 +73,8 @@ static struct trace_guest *add_guest(unsigned int cid, const char *name)
 	guest->cid = cid;
 	guest->pid = -1;
 
+	find_pid_by_cid(guest);
+
 	return guest;
 }
 
@@ -307,11 +311,8 @@ struct trace_guest *trace_get_guest(unsigned int cid, const char *name)
 
 	if (cid > 0) {
 		guest = get_guest_by_cid(cid);
-		if (!guest && name) {
+		if (!guest && name)
 			guest = add_guest(cid, name);
-			if (guest)
-				find_pid_by_cid(guest);
-		}
 	}
 	return guest;
 }
@@ -321,7 +322,6 @@ struct trace_guest *trace_get_guest(unsigned int cid, const char *name)
 #define VM_CID_ID	"address='"
 static void read_guest_cid(char *name)
 {
-	struct trace_guest *guest;
 	char *cmd = NULL;
 	char line[512];
 	char *cid;
@@ -343,9 +343,7 @@ static void read_guest_cid(char *name)
 		cid_id = strtol(cid + strlen(VM_CID_ID), NULL, 10);
 		if ((cid_id == INT_MIN || cid_id == INT_MAX) && errno == ERANGE)
 			continue;
-		guest = add_guest(cid_id, name);
-		if (guest)
-			find_pid_by_cid(guest);
+		add_guest(cid_id, name);
 		break;
 	}
 
-- 
2.35.1

[PATCH 05/26] trace-cmd: Move find_tasks() into add_guest()

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

In order to simplify the logic a bit, move the find_tasks() code (to find
the PIDs on the host that map to the guest) into the add_guest() function
where the guest is first added.

Link: https://lore.kernel.org/linux-trace-devel/20220428150635.789051-4-rostedt@goodmis.org

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/trace-record.c | 35 -----------------------------------
 tracecmd/trace-vm.c     | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index 67ad1208f90c..5f7ac15dd80a 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -3215,38 +3215,6 @@ static int do_accept(int sd)
 	return -1;
 }
 
-/* Find all the tasks associated with the guest pid */
-static void find_tasks(struct trace_guest *guest)
-{
-	struct dirent *dent;
-	char *path;
-	DIR *dir;
-	int ret;
-	int tasks = 0;
-
-	ret = asprintf(&path, "/proc/%d/task", guest->pid);
-	if (ret < 0)
-		return;
-
-	dir = opendir(path);
-	free(path);
-	if (!dir)
-		return;
-
-	while ((dent = readdir(dir))) {
-		int *pids;
-		if (!(dent->d_type == DT_DIR && is_digits(dent->d_name)))
-			continue;
-		pids = realloc(guest->task_pids, sizeof(int) * (tasks + 2));
-		if (!pids)
-			break;
-		pids[tasks++] = strtol(dent->d_name, NULL, 0);
-		pids[tasks] = -1;
-		guest->task_pids = pids;
-	}
-	closedir(dir);
-}
-
 static char *parse_guest_name(char *gname, int *cid, int *port,
 			      struct addrinfo **res)
 {
@@ -3287,9 +3255,6 @@ static char *parse_guest_name(char *gname, int *cid, int *port,
 		guest = trace_get_guest(*cid, gname);
 	if (guest) {
 		*cid = guest->cid;
-		/* Mapping not found, search for them */
-		if (!guest->cpu_pid)
-			find_tasks(guest);
 		return guest->name;
 	}
 
diff --git a/tracecmd/trace-vm.c b/tracecmd/trace-vm.c
index 4b4e039bd2cb..84f28824e3c5 100644
--- a/tracecmd/trace-vm.c
+++ b/tracecmd/trace-vm.c
@@ -54,6 +54,38 @@ bool trace_have_guests_pid(void)
 	return true;
 }
 
+/* Find all the tasks associated with the guest pid */
+static void find_tasks(struct trace_guest *guest)
+{
+	struct dirent *dent;
+	char *path;
+	DIR *dir;
+	int ret;
+	int tasks = 0;
+
+	ret = asprintf(&path, "/proc/%d/task", guest->pid);
+	if (ret < 0)
+		return;
+
+	dir = opendir(path);
+	free(path);
+	if (!dir)
+		return;
+
+	while ((dent = readdir(dir))) {
+		int *pids;
+		if (!(dent->d_type == DT_DIR && is_digits(dent->d_name)))
+			continue;
+		pids = realloc(guest->task_pids, sizeof(int) * (tasks + 2));
+		if (!pids)
+			break;
+		pids[tasks++] = strtol(dent->d_name, NULL, 0);
+		pids[tasks] = -1;
+		guest->task_pids = pids;
+	}
+	closedir(dir);
+}
+
 static void find_pid_by_cid(struct trace_guest *guest);
 
 static struct trace_guest *add_guest(unsigned int cid, const char *name)
@@ -74,6 +106,7 @@ static struct trace_guest *add_guest(unsigned int cid, const char *name)
 	guest->pid = -1;
 
 	find_pid_by_cid(guest);
+	find_tasks(guest);
 
 	return guest;
 }
-- 
2.35.1

[PATCH 06/26] trace-cmd: Move trace_msg cache file to memfd

[Steven Rostedt]

From: Joel Fernandes <joel@joelfernandes.org>

This is cleaner and also fixes issues in Android where /tmp does not
exist.

With this, host-guest tracing works on Android VM running on ChromeOS
Linux.

Link: https://lore.kernel.org/linux-trace-devel/20220405192204.1551283-1-joel@joelfernandes.org

Signed-off-by: Joel Fernandes <joel@joelfernandes.org>
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 lib/trace-cmd/include/private/trace-cmd-private.h | 2 --
 lib/trace-cmd/trace-msg.c                         | 5 ++---
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/lib/trace-cmd/include/private/trace-cmd-private.h b/lib/trace-cmd/include/private/trace-cmd-private.h
index 766e0a762c2b..0efc2a1c4850 100644
--- a/lib/trace-cmd/include/private/trace-cmd-private.h
+++ b/lib/trace-cmd/include/private/trace-cmd-private.h
@@ -381,7 +381,6 @@ enum tracecmd_msg_flags {
 };
 
 /* for both client and server */
-#define MSG_CACHE_FILE "/tmp/trace_msg_cacheXXXXXX"
 struct tracecmd_msg_handle {
 	int			fd;
 	short			cpu_count;
@@ -390,7 +389,6 @@ struct tracecmd_msg_handle {
 	bool			done;
 	bool			cache;
 	int			cfd;
-	char			cfile[sizeof(MSG_CACHE_FILE)];
 };
 
 struct tracecmd_tsync_protos {
diff --git a/lib/trace-cmd/trace-msg.c b/lib/trace-cmd/trace-msg.c
index 39465ade8ab3..5ba2eeb2d183 100644
--- a/lib/trace-cmd/trace-msg.c
+++ b/lib/trace-cmd/trace-msg.c
@@ -19,6 +19,7 @@
 #include <string.h>
 #include <unistd.h>
 #include <arpa/inet.h>
+#include <sys/mman.h>
 #include <sys/types.h>
 #include <linux/types.h>
 
@@ -590,11 +591,9 @@ tracecmd_msg_handle_alloc(int fd, unsigned long flags)
 int tracecmd_msg_handle_cache(struct tracecmd_msg_handle *msg_handle)
 {
 	if (msg_handle->cfd < 0) {
-		strcpy(msg_handle->cfile, MSG_CACHE_FILE);
-		msg_handle->cfd = mkstemp(msg_handle->cfile);
+		msg_handle->cfd = memfd_create("trace_msg_cache", 0);
 		if (msg_handle->cfd < 0)
 			return -1;
-		unlink(msg_handle->cfile);
 	}
 	msg_handle->cache = true;
 	return 0;
-- 
2.35.1

[PATCH 07/26] trace-cmd Makefile: Change test-build to link as well

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

Remove the '-c' option from the test-build macro build. To test if a
function exists, it needs to go through the link phase, otherwise it can
pass with an undefined symbol.

Link: https://lore.kernel.org/linux-trace-devel/20220428174949.797657-2-rostedt@goodmis.org

Cc: Joel Fernandes <joel@joelfernandes.org>
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 982514baad01..9fbc6de25d05 100644
--- a/Makefile
+++ b/Makefile
@@ -163,7 +163,7 @@ export NO_PYTHON
 # $(call test-build, snippet, ret) -> ret if snippet compiles
 #                                  -> empty otherwise
 test-build = $(if $(shell sh -c 'echo "$(1)" | \
-	$(CC) -o /dev/null -c -x c - > /dev/null 2>&1 && echo y'), $2)
+	$(CC) -o /dev/null -x c - > /dev/null 2>&1 && echo y'), $2)
 
 UDIS86_AVAILABLE := $(call test-build,\#include <udis86.h>, y)
 ifneq ($(strip $(UDIS86_AVAILABLE)), y)
-- 
2.35.1

[PATCH 08/26] trace-cmd agent: Test if memfd_create() is available

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

Some systems do not have memfd_create() available and it is a regression
to require it. Fall back to tempfs if memfd_create() is not available at
build time.

Link: https://lore.kernel.org/linux-trace-devel/20220428174949.797657-3-rostedt@goodmis.org

Cc: Joel Fernandes <joel@joelfernandes.org>
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 Makefile                                          | 11 ++++++++++-
 lib/trace-cmd/include/private/trace-cmd-private.h |  5 +++++
 lib/trace-cmd/trace-msg.c                         |  8 ++++++++
 3 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 9fbc6de25d05..abc4ac723db4 100644
--- a/Makefile
+++ b/Makefile
@@ -184,6 +184,15 @@ endef
 # have flush/fua block layer instead of barriers?
 blk-flags := $(call test-build,$(BLK_TC_FLUSH_SOURCE),-DHAVE_BLK_TC_FLUSH)
 
+define MEMFD_CREATE_SOURCE
+#define _GNU_SOURCE
+#include <sys/mman.h>
+int main(void) { return memfd_create(\"test\", 0); }
+endef
+
+# have memfd_create available
+memfd-flags := $(call test-build,$(MEMFD_CREATE_SOURCE),-DHAVE_MEMFD_CREATE)
+
 ifeq ("$(origin O)", "command line")
 
   saved-output := $(O)
@@ -363,7 +372,7 @@ endif
 # Append required CFLAGS
 override CFLAGS += $(INCLUDES) $(VAR_DIR)
 override CFLAGS += $(PLUGIN_DIR_TRACECMD_SQ)
-override CFLAGS += $(udis86-flags) $(blk-flags)
+override CFLAGS += $(udis86-flags) $(blk-flags) $(memfd-flags)
 override LDFLAGS += $(udis86-ldflags)
 
 CMD_TARGETS = trace-cmd $(BUILD_PYTHON)
diff --git a/lib/trace-cmd/include/private/trace-cmd-private.h b/lib/trace-cmd/include/private/trace-cmd-private.h
index 0efc2a1c4850..3b84170f2258 100644
--- a/lib/trace-cmd/include/private/trace-cmd-private.h
+++ b/lib/trace-cmd/include/private/trace-cmd-private.h
@@ -380,6 +380,8 @@ enum tracecmd_msg_flags {
 	TRACECMD_MSG_FL_USE_VSOCK	= 1 << 1,
 };
 
+#define MSG_CACHE_FILE "/tmp/trace_msg_cacheXXXXXX"
+
 /* for both client and server */
 struct tracecmd_msg_handle {
 	int			fd;
@@ -389,6 +391,9 @@ struct tracecmd_msg_handle {
 	bool			done;
 	bool			cache;
 	int			cfd;
+#ifndef HAVE_MEMFD_CREATE
+	char			cfile[sizeof(MSG_CACHE_FILE)];
+#endif
 };
 
 struct tracecmd_tsync_protos {
diff --git a/lib/trace-cmd/trace-msg.c b/lib/trace-cmd/trace-msg.c
index 5ba2eeb2d183..9899fa8796e2 100644
--- a/lib/trace-cmd/trace-msg.c
+++ b/lib/trace-cmd/trace-msg.c
@@ -591,9 +591,17 @@ tracecmd_msg_handle_alloc(int fd, unsigned long flags)
 int tracecmd_msg_handle_cache(struct tracecmd_msg_handle *msg_handle)
 {
 	if (msg_handle->cfd < 0) {
+#ifdef HAVE_MEMFD_CREATE
 		msg_handle->cfd = memfd_create("trace_msg_cache", 0);
 		if (msg_handle->cfd < 0)
 			return -1;
+#else
+		strcpy(msg_handle->cfile, MSG_CACHE_FILE);
+		msg_handle->cfd = mkstemp(msg_handle->cfile);
+		if (msg_handle->cfd < 0)
+			return -1;
+		unlink(msg_handle->cfile);
+#endif
 	}
 	msg_handle->cache = true;
 	return 0;
-- 
2.35.1

[PATCH 09/26] trace-cmd: Add kernel-doc to trace_record_agent()

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

I'm getting confused by what exactly is being passed to
trace_record_agent(). Document it, so I can at least be only temporarily
confused.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/trace-record.c | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index 5f7ac15dd80a..8b6b27595c96 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -7192,6 +7192,24 @@ void trace_record(int argc, char **argv)
 	exit(0);
 }
 
+/**
+ * trace_record_agent - record command running from the agent
+ * @msg_handle: The handle to communicate with the peer
+ * @cpus: The number of CPUs the agent has to record
+ * @fds: The array of file descriptors for the CPUs
+ * @argc: The number of arguments to pass to the record session
+ * @argv: The arguments to pass to the record session
+ * @use_fifos: True if fifos are used instead of sockets.
+ * @trace_id: The agent's trace_id
+ * @host: Set if this is an IP connection and not a vsocket one
+ *
+ * This is used to enable tracing via the record command just
+ * like trace-cmd record, but it is being done via the agent
+ * and all the data is being transfered to the peer that is
+ * connected on the other end of the sockets.
+ *
+ *  Returns zero on success, negative otherwise.
+ */
 int trace_record_agent(struct tracecmd_msg_handle *msg_handle,
 		       int cpus, int *fds,
 		       int argc, char **argv,
-- 
2.35.1

[PATCH 10/26] trace-cmd: Move selecting tsync protocol out of tracecmd_tsync_with_host()

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

In preparation of the agent proxy, separate out the decision of what
protocol is used for time synchronization out of the communication with
the host. It should be done separately.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 .../include/private/trace-cmd-private.h       |  9 ++--
 lib/trace-cmd/trace-timesync.c                | 51 ++++++++-----------
 tracecmd/trace-agent.c                        | 13 ++---
 3 files changed, 31 insertions(+), 42 deletions(-)

diff --git a/lib/trace-cmd/include/private/trace-cmd-private.h b/lib/trace-cmd/include/private/trace-cmd-private.h
index 3b84170f2258..77b387b494ee 100644
--- a/lib/trace-cmd/include/private/trace-cmd-private.h
+++ b/lib/trace-cmd/include/private/trace-cmd-private.h
@@ -497,9 +497,8 @@ void tracecmd_tsync_init(void);
 int tracecmd_tsync_proto_getall(struct tracecmd_tsync_protos **protos, const char *clock, int role);
 bool tsync_proto_is_supported(const char *proto_name);
 struct tracecmd_time_sync *
-tracecmd_tsync_with_host(int fd,
-			 const struct tracecmd_tsync_protos *tsync_protos,
-			 const char *clock, int remote_id, int local_id);
+tracecmd_tsync_with_host(int fd, const char *proto, const char *clock,
+			 int remote_id, int local_id);
 int tracecmd_tsync_with_host_stop(struct tracecmd_time_sync *tsync);
 struct tracecmd_time_sync *
 tracecmd_tsync_with_guest(unsigned long long trace_id, int loop_interval,
@@ -509,8 +508,8 @@ int tracecmd_tsync_with_guest_stop(struct tracecmd_time_sync *tsync);
 int tracecmd_tsync_get_offsets(struct tracecmd_time_sync *tsync, int cpu,
 			       int *count, long long **ts,
 			       long long **offsets, long long **scalings, long long **frac);
-int tracecmd_tsync_get_selected_proto(struct tracecmd_time_sync *tsync,
-				      char **selected_proto);
+const char *tracecmd_tsync_get_proto(const struct tracecmd_tsync_protos *protos,
+			 const char *clock);
 void tracecmd_tsync_free(struct tracecmd_time_sync *tsync);
 int tracecmd_write_guest_time_shift(struct tracecmd_output *handle,
 				    struct tracecmd_time_sync *tsync);
diff --git a/lib/trace-cmd/trace-timesync.c b/lib/trace-cmd/trace-timesync.c
index cc44af38e8ad..966aa56c5cc0 100644
--- a/lib/trace-cmd/trace-timesync.c
+++ b/lib/trace-cmd/trace-timesync.c
@@ -239,6 +239,23 @@ tsync_proto_select(const struct tracecmd_tsync_protos *protos,
 	return NULL;
 }
 
+/**
+ * tracecmd_tsync_get_proto - return the appropriate synchronization protocol
+ * @protos: The list of synchronization protocols to choose from
+ * @clock: The clock that is being used (or NULL for unknown).
+ *
+ * Retuns pointer to a protocol name, that can be used with the peer, or NULL
+ *	  in case there is no match with supported protocols.
+ *	  The returned string MUST NOT be freed by the caller
+ */
+__hidden const char *
+tracecmd_tsync_get_proto(const struct tracecmd_tsync_protos *protos,
+			 const char *clock)
+{
+	return tsync_proto_select(protos, clock,
+				  TRACECMD_TIME_SYNC_ROLE_GUEST);
+}
+
 /**
  * tracecmd_tsync_proto_getall - Returns list of all supported
  *				 time sync protocols
@@ -948,7 +965,7 @@ out:
 /**
  * tracecmd_tsync_with_host - Synchronize timestamps with host
  * @fd: File descriptor connecting with the host
- * @tsync_protos: List of tsync protocols, supported by the host
+ * @proto: The selected protocol
  * @clock: Trace clock, used for that session
  * @port: returned, VSOCKET port, on which the guest listens for tsync requests
  * @remote_id: Identifier to uniquely identify the remote host
@@ -961,25 +978,19 @@ out:
  * until tracecmd_tsync_with_host_stop() is called.
  */
 struct tracecmd_time_sync *
-tracecmd_tsync_with_host(int fd,
-			 const struct tracecmd_tsync_protos *tsync_protos,
-			 const char *clock, int remote_id, int local_id)
+tracecmd_tsync_with_host(int fd, const char *proto, const char *clock,
+			 int remote_id, int local_id)
 {
 	struct tracecmd_time_sync *tsync;
 	cpu_set_t *pin_mask = NULL;
 	pthread_attr_t attrib;
 	size_t mask_size = 0;
-	const char *proto;
 	int ret;
 
 	tsync = calloc(1, sizeof(struct tracecmd_time_sync));
 	if (!tsync)
 		return NULL;
 
-	proto = tsync_proto_select(tsync_protos, clock,
-				   TRACECMD_TIME_SYNC_ROLE_GUEST);
-	if (!proto)
-		goto error;
 	tsync->proto_name = strdup(proto);
 	tsync->msg_handle = tracecmd_msg_handle_alloc(fd, 0);
 	if (clock)
@@ -1033,25 +1044,3 @@ int tracecmd_tsync_with_host_stop(struct tracecmd_time_sync *tsync)
 {
 	return pthread_join(tsync->thread, NULL);
 }
-
-/**
- * tracecmd_tsync_get_selected_proto - Return the seleceted time sync protocol
- * @tsync: Time sync context, representing a running time sync session
- * @selected_proto: return, name of the selected time sync protocol for this session
- *
- * Returns 0 on success, or -1 in case of an error.
- *
- */
-int tracecmd_tsync_get_selected_proto(struct tracecmd_time_sync *tsync,
-				      char **selected_proto)
-{
-	if (!tsync)
-		return -1;
-
-	if (selected_proto) {
-		if (!tsync->proto_name)
-			return -1;
-		(*selected_proto) = strdup(tsync->proto_name);
-	}
-	return 0;
-}
diff --git a/tracecmd/trace-agent.c b/tracecmd/trace-agent.c
index f0723a6601a4..2fe31f71e47a 100644
--- a/tracecmd/trace-agent.c
+++ b/tracecmd/trace-agent.c
@@ -127,7 +127,7 @@ static void agent_handle(int sd, int nr_cpus, int page_size, const char *network
 	struct tracecmd_tsync_protos *tsync_protos = NULL;
 	struct tracecmd_time_sync *tsync = NULL;
 	struct tracecmd_msg_handle *msg_handle;
-	char *tsync_proto = NULL;
+	const char *tsync_proto = NULL;
 	unsigned long long trace_id;
 	unsigned int remote_id;
 	unsigned int local_id;
@@ -155,12 +155,15 @@ static void agent_handle(int sd, int nr_cpus, int page_size, const char *network
 	if (ret < 0)
 		die("Failed to receive trace request");
 
+	tsync_proto = tracecmd_tsync_get_proto(tsync_protos,
+					       get_clock(argc, argv));
+
 	if (use_fifos && open_agent_fifos(nr_cpus, fds))
 		use_fifos = false;
 
 	if (!use_fifos)
 		make_sockets(nr_cpus, fds, ports, network);
-	if (tsync_protos && tsync_protos->names) {
+	if (tsync_proto) {
 		if (network) {
 			/* For now just use something */
 			remote_id = 2;
@@ -184,13 +187,11 @@ static void agent_handle(int sd, int nr_cpus, int page_size, const char *network
 			}
 		}
 		if (fd >= 0) {
-			tsync = tracecmd_tsync_with_host(fd, tsync_protos,
+			tsync = tracecmd_tsync_with_host(fd, tsync_proto,
 							 get_clock(argc, argv),
 							 remote_id, local_id);
 		}
-		if (tsync) {
-			tracecmd_tsync_get_selected_proto(tsync, &tsync_proto);
-		} else {
+		if (!tsync) {
 			warning("Failed to negotiate timestamps synchronization with the host");
 			if (fd >= 0)
 				close(fd);
-- 
2.35.1

[PATCH 11/26] trace-cmd: Move accepting tsync connection out of tracecmd_tsync_with_host()

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

In preparation for adding a agent proxy, move the acceptance of the tsync
socket out of tracecmd_tsync_with_host(). This will allow the agent to do
the accept and then act like a host.

A side effect of this is that currently if the host fails to connect to
the agent for time synchronization, the thread will never continue and be
stuck at the "accept()". This will also hang the agent when it tries to
join that thread. The recording on the host side would work as normal, but
this would leave the agent process stuck, and this could cause a leak of
processes.

By accepting before the sync, the agent would not continue, and this would
also make the host side fail (this is a good thing), and then this issue
will be detected right at the beginning.

This also requires moving the tracecmd_msg_send_trace_resp() before the
synchronization, otherwise there would be a deadlock (with the agent
waiting for the record to connect, and the record waiting for a response
from the agent).

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 lib/trace-cmd/trace-timesync.c | 21 +----------------
 tracecmd/trace-agent.c         | 41 +++++++++++++++++++++++++++-------
 2 files changed, 34 insertions(+), 28 deletions(-)

diff --git a/lib/trace-cmd/trace-timesync.c b/lib/trace-cmd/trace-timesync.c
index 966aa56c5cc0..e192a74a1974 100644
--- a/lib/trace-cmd/trace-timesync.c
+++ b/lib/trace-cmd/trace-timesync.c
@@ -938,28 +938,9 @@ int tracecmd_tsync_with_guest_stop(struct tracecmd_time_sync *tsync)
 static void *tsync_agent_thread(void *data)
 {
 	struct tracecmd_time_sync *tsync = data;
-	long ret = 0;
-	int sd;
-
-	while (true) {
-		tracecmd_debug("Listening on fd:%d\n", tsync->msg_handle->fd);
-		sd = accept(tsync->msg_handle->fd, NULL, NULL);
-		tracecmd_debug("Accepted fd:%d\n", sd);
-		if (sd < 0) {
-			if (errno == EINTR)
-				continue;
-			ret = -1;
-			goto out;
-		}
-		break;
-	}
-	close(tsync->msg_handle->fd);
-	tsync->msg_handle->fd = sd;
 
 	tsync_with_host(tsync);
-
-out:
-	pthread_exit((void *)ret);
+	pthread_exit(NULL);
 }
 
 /**
diff --git a/tracecmd/trace-agent.c b/tracecmd/trace-agent.c
index 2fe31f71e47a..7ee5fc8352c6 100644
--- a/tracecmd/trace-agent.c
+++ b/tracecmd/trace-agent.c
@@ -122,6 +122,28 @@ static void trace_print_connection(int fd, const char *network)
 		tracecmd_debug("Could not print connection fd:%d\n", fd);
 }
 
+static int wait_for_connection(int fd)
+{
+	int sd;
+
+	if (fd < 0)
+		return -1;
+
+	while (true) {
+		tracecmd_debug("Listening on fd:%d\n", fd);
+		sd = accept(fd, NULL, NULL);
+		tracecmd_debug("Accepted fd:%d\n", sd);
+		if (sd < 0) {
+			if (errno == EINTR)
+				continue;
+			return -1;
+		}
+		break;
+	}
+	close(fd);
+	return sd;
+}
+
 static void agent_handle(int sd, int nr_cpus, int page_size, const char *network)
 {
 	struct tracecmd_tsync_protos *tsync_protos = NULL;
@@ -186,23 +208,26 @@ static void agent_handle(int sd, int nr_cpus, int page_size, const char *network
 				fd = -1;
 			}
 		}
-		if (fd >= 0) {
+	}
+	trace_id = tracecmd_generate_traceid();
+	ret = tracecmd_msg_send_trace_resp(msg_handle, nr_cpus, page_size,
+					   ports, use_fifos, trace_id,
+					   tsync_proto, tsync_port);
+	if (ret < 0)
+		die("Failed to send trace response");
+
+	if (tsync_proto) {
+		fd = wait_for_connection(fd);
+		if (fd >= 0)
 			tsync = tracecmd_tsync_with_host(fd, tsync_proto,
 							 get_clock(argc, argv),
 							 remote_id, local_id);
-		}
 		if (!tsync) {
 			warning("Failed to negotiate timestamps synchronization with the host");
 			if (fd >= 0)
 				close(fd);
 		}
 	}
-	trace_id = tracecmd_generate_traceid();
-	ret = tracecmd_msg_send_trace_resp(msg_handle, nr_cpus, page_size,
-					   ports, use_fifos, trace_id,
-					   tsync_proto, tsync_port);
-	if (ret < 0)
-		die("Failed to send trace response");
 
 	trace_record_agent(msg_handle, nr_cpus, fds, argc, argv,
 			   use_fifos, trace_id, network);
-- 
2.35.1

[PATCH 12/26] trace-cmd: Have get_vsocket_params() cid and rcid parameters be optional

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

In order to retrieve just the remote cid that the agent proxy will need to
do, make the parameters to get_vsocket_params() optional, where a NULL can
be passed in and it will only update the non-NULL parameters.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/trace-vsock.c | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/tracecmd/trace-vsock.c b/tracecmd/trace-vsock.c
index 39294e7a2a3c..3bad9efad39a 100644
--- a/tracecmd/trace-vsock.c
+++ b/tracecmd/trace-vsock.c
@@ -76,20 +76,24 @@ int get_vsocket_params(int fd, unsigned int *lcid, unsigned int *rcid)
 	struct sockaddr_vm addr;
 	socklen_t addr_len = sizeof(addr);
 
-	memset(&addr, 0, sizeof(addr));
-	if (getsockname(fd, (struct sockaddr *)&addr, &addr_len))
-		return -1;
-	if (addr.svm_family != AF_VSOCK)
-		return -1;
-	*lcid = addr.svm_cid;
-
-	memset(&addr, 0, sizeof(addr));
-	addr_len = sizeof(addr);
-	if (getpeername(fd, (struct sockaddr *)&addr, &addr_len))
-		return -1;
-	if (addr.svm_family != AF_VSOCK)
-		return -1;
-	*rcid = addr.svm_cid;
+	if (lcid) {
+		memset(&addr, 0, sizeof(addr));
+		if (getsockname(fd, (struct sockaddr *)&addr, &addr_len))
+			return -1;
+		if (addr.svm_family != AF_VSOCK)
+			return -1;
+		*lcid = addr.svm_cid;
+	}
+
+	if (rcid) {
+		memset(&addr, 0, sizeof(addr));
+		addr_len = sizeof(addr);
+		if (getpeername(fd, (struct sockaddr *)&addr, &addr_len))
+			return -1;
+		if (addr.svm_family != AF_VSOCK)
+			return -1;
+		*rcid = addr.svm_cid;
+	}
 
 	return 0;
 }
-- 
2.35.1

[PATCH 13/26] trace-cmd agent: Add trace_tsync_as_guest() helper function

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

In preparation to implementing a agent proxy, where the record side will
need to synchronize as the guest, pull out the code that does so into its
own function. It will later be placed in a separate file.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/trace-agent.c | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/tracecmd/trace-agent.c b/tracecmd/trace-agent.c
index 7ee5fc8352c6..d226833b4fd0 100644
--- a/tracecmd/trace-agent.c
+++ b/tracecmd/trace-agent.c
@@ -144,6 +144,23 @@ static int wait_for_connection(int fd)
 	return sd;
 }
 
+static struct tracecmd_time_sync *
+trace_tsync_as_guest(int fd, const char *tsync_proto, const char *clock,
+	       unsigned int remote_id, unsigned int local_id)
+{
+	struct tracecmd_time_sync *tsync;
+
+	if (fd < 0)
+		return NULL;
+
+	tsync = tracecmd_tsync_with_host(fd, tsync_proto, clock,
+					 remote_id, local_id);
+	if (!tsync)
+		warning("Failed to negotiate timestamps synchronization with the host");
+
+	return tsync;
+}
+
 static void agent_handle(int sd, int nr_cpus, int page_size, const char *network)
 {
 	struct tracecmd_tsync_protos *tsync_protos = NULL;
@@ -218,15 +235,10 @@ static void agent_handle(int sd, int nr_cpus, int page_size, const char *network
 
 	if (tsync_proto) {
 		fd = wait_for_connection(fd);
-		if (fd >= 0)
-			tsync = tracecmd_tsync_with_host(fd, tsync_proto,
-							 get_clock(argc, argv),
-							 remote_id, local_id);
-		if (!tsync) {
-			warning("Failed to negotiate timestamps synchronization with the host");
-			if (fd >= 0)
-				close(fd);
-		}
+		tsync = trace_tsync_as_guest(fd, tsync_proto, get_clock(argc, argv),
+					     remote_id, local_id);
+		if (!tsync)
+			close(fd);
 	}
 
 	trace_record_agent(msg_handle, nr_cpus, fds, argc, argv,
-- 
2.35.1

[PATCH 14/26] trace-cmd record: Pass cpu_count instead of an instance to stop_mapping_vcpus()

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

The stop_mapping_vcpus() takes an instance as a parameter just to get to
the cpu_count. There's no reason to not just pass the cpu_count instead.
This will also simplify the code when adding an agent proxy that will use
this code as well.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/trace-record.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index 8b6b27595c96..f05ad09c9723 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -3920,8 +3920,7 @@ static int map_vcpus(struct tep_event *event, struct tep_record *record,
 	return i == tmap->max_cpus;
 }
 
-static void stop_mapping_vcpus(struct buffer_instance *instance,
-			       struct trace_guest *guest)
+static void stop_mapping_vcpus(int cpu_count, struct trace_guest *guest)
 {
 	struct trace_mapping tmap = { };
 	struct tep_handle *tep;
@@ -3932,7 +3931,7 @@ static void stop_mapping_vcpus(struct buffer_instance *instance,
 		return;
 
 	tmap.pids = guest->task_pids;
-	tmap.max_cpus = instance->cpu_count;
+	tmap.max_cpus = cpu_count;
 
 	tmap.map = malloc(sizeof(*tmap.map) * tmap.max_cpus);
 	if (!tmap.map)
@@ -4010,7 +4009,7 @@ static int host_tsync(struct common_record_context *ctx,
 						    instance->cpu_count,
 						    proto, ctx->clock);
 	if (!is_network(instance))
-		stop_mapping_vcpus(instance, guest);
+		stop_mapping_vcpus(instance->cpu_count, guest);
 
 	if (!instance->tsync)
 		return -1;
-- 
2.35.1

[PATCH 15/26] trace-cmd record: Add trace_tsync_as_host() helper

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

In the process of adding an agent proxy, it will be useful to have a
function that will allow the agent to synchronize as the host. Move the
code to do so into a helper function that can be used by the agent. This
will be later moved to a separate file.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/trace-record.c | 57 +++++++++++++++++++++++++++--------------
 1 file changed, 38 insertions(+), 19 deletions(-)

diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index f05ad09c9723..2c595a7918f0 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -3979,12 +3979,42 @@ static void stop_mapping_vcpus(int cpu_count, struct trace_guest *guest)
 	tracefs_instance_free(guest->instance);
 }
 
+static struct tracecmd_time_sync *
+trace_tsync_as_host(int fd, unsigned long long trace_id,
+		    int loop_interval, int guest_id,
+		    int guest_cpus, const char *proto_name,
+		    const char *clock)
+{
+	struct tracecmd_time_sync *tsync;
+	struct trace_guest *guest;
+	int guest_pid = -1;
+
+	if (fd < 0)
+		return NULL;
+
+	if (guest_id >= 0) {
+		guest = trace_get_guest(guest_id, NULL);
+		if (guest == NULL)
+			return NULL;
+		guest_pid = guest->pid;
+		start_mapping_vcpus(guest);
+	}
+
+	tsync = tracecmd_tsync_with_guest(trace_id, loop_interval, fd,
+					  guest_pid, guest_cpus, proto_name,
+					  clock);
+
+	if (guest_id >= 0)
+		stop_mapping_vcpus(guest_cpus, guest);
+
+	return tsync;
+}
+
 static int host_tsync(struct common_record_context *ctx,
 		      struct buffer_instance *instance,
 		      unsigned int tsync_port, char *proto)
 {
-	struct trace_guest *guest;
-	int guest_pid = -1;
+	int guest_id = -1;
 	int fd;
 
 	if (!proto)
@@ -3994,27 +4024,16 @@ static int host_tsync(struct common_record_context *ctx,
 		fd = connect_port(instance->name, tsync_port,
 				  instance->port_type);
 	} else {
-		guest = trace_get_guest(instance->cid, NULL);
-		if (guest == NULL)
-			return -1;
-
-		guest_pid = guest->pid;
-		start_mapping_vcpus(guest);
+		guest_id = instance->cid;
 		fd = trace_vsock_open(instance->cid, tsync_port);
 	}
 
-	instance->tsync = tracecmd_tsync_with_guest(top_instance.trace_id,
-						    instance->tsync_loop_interval,
-						    fd, guest_pid,
-						    instance->cpu_count,
-						    proto, ctx->clock);
-	if (!is_network(instance))
-		stop_mapping_vcpus(instance->cpu_count, guest);
+	instance->tsync = trace_tsync_as_host(fd, top_instance.trace_id,
+					      instance->tsync_loop_interval,
+					      guest_id, instance->cpu_count,
+					      proto, ctx->clock);
 
-	if (!instance->tsync)
-		return -1;
-
-	return 0;
+	return instance->tsync ? 0 : -1;
 }
 
 static void connect_to_agent(struct common_record_context *ctx,
-- 
2.35.1

[PATCH 16/26] trace-cmd: Move tsync as guest and host helpers into trace-tsync.c

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

Create a trace-tsync.c file to house the tsync host and guest helper
functions, and have the trace-record.c and trace-agent.c call them.

This is just a clean up to simplify the code for when an agent proxy is
added.

Also add some comments about these functions.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/Makefile              |   1 +
 tracecmd/include/trace-local.h |  10 ++
 tracecmd/trace-agent.c         |  17 ---
 tracecmd/trace-record.c        | 186 -------------------------
 tracecmd/trace-tsync.c         | 242 +++++++++++++++++++++++++++++++++
 5 files changed, 253 insertions(+), 203 deletions(-)
 create mode 100644 tracecmd/trace-tsync.c

diff --git a/tracecmd/Makefile b/tracecmd/Makefile
index 0114948fe385..56be9a3b88d1 100644
--- a/tracecmd/Makefile
+++ b/tracecmd/Makefile
@@ -38,6 +38,7 @@ TRACE_CMD_OBJS += trace-clear.o
 TRACE_CMD_OBJS += trace-vm.o
 TRACE_CMD_OBJS += trace-convert.o
 TRACE_CMD_OBJS += trace-agent.o
+TRACE_CMD_OBJS += trace-tsync.o
 TRACE_CMD_OBJS += trace-setup-guest.o
 ifeq ($(VSOCK_DEFINED), 1)
 TRACE_CMD_OBJS += trace-vsock.o
diff --git a/tracecmd/include/trace-local.h b/tracecmd/include/trace-local.h
index f3b805fa7bad..5febe9a60a54 100644
--- a/tracecmd/include/trace-local.h
+++ b/tracecmd/include/trace-local.h
@@ -341,6 +341,16 @@ int get_guest_pid(unsigned int guest_cid);
 int get_guest_vcpu_pid(unsigned int guest_cid, unsigned int guest_vcpu);
 void trace_add_guest_info(struct tracecmd_output *handle, struct buffer_instance *instance);
 
+struct tracecmd_time_sync *
+trace_tsync_as_host(int fd, unsigned long long trace_id,
+		    int loop_interval, int guest_id,
+		    int guest_cpus, const char *proto_name,
+		    const char *clock);
+
+struct tracecmd_time_sync *
+trace_tsync_as_guest(int fd, const char *tsync_proto, const char *clock,
+	       unsigned int remote_id, unsigned int local_id);
+
 /* moved from trace-cmd.h */
 void tracecmd_remove_instances(void);
 int tracecmd_add_event(const char *event_str, int stack);
diff --git a/tracecmd/trace-agent.c b/tracecmd/trace-agent.c
index d226833b4fd0..920094702b6d 100644
--- a/tracecmd/trace-agent.c
+++ b/tracecmd/trace-agent.c
@@ -144,23 +144,6 @@ static int wait_for_connection(int fd)
 	return sd;
 }
 
-static struct tracecmd_time_sync *
-trace_tsync_as_guest(int fd, const char *tsync_proto, const char *clock,
-	       unsigned int remote_id, unsigned int local_id)
-{
-	struct tracecmd_time_sync *tsync;
-
-	if (fd < 0)
-		return NULL;
-
-	tsync = tracecmd_tsync_with_host(fd, tsync_proto, clock,
-					 remote_id, local_id);
-	if (!tsync)
-		warning("Failed to negotiate timestamps synchronization with the host");
-
-	return tsync;
-}
-
 static void agent_handle(int sd, int nr_cpus, int page_size, const char *network)
 {
 	struct tracecmd_tsync_protos *tsync_protos = NULL;
diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index 2c595a7918f0..bf85e7936b73 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -3824,192 +3824,6 @@ static int open_guest_fifos(const char *guest, int **fds)
 	return i;
 }
 
-struct trace_mapping {
-	struct tep_event		*kvm_entry;
-	struct tep_format_field		*vcpu_id;
-	struct tep_format_field		*common_pid;
-	int				*pids;
-	int				*map;
-	int				max_cpus;
-};
-
-static void start_mapping_vcpus(struct trace_guest *guest)
-{
-	char *pids = NULL;
-	char *t;
-	int len = 0;
-	int s;
-	int i;
-
-	if (!guest->task_pids)
-		return;
-
-	guest->instance = tracefs_instance_create("map_guest_pids");
-	if (!guest->instance)
-		return;
-
-	for (i = 0; guest->task_pids[i] >= 0; i++) {
-		s = snprintf(NULL, 0, "%d ", guest->task_pids[i]);
-		t = realloc(pids, len + s + 1);
-		if (!t) {
-			free(pids);
-			pids = NULL;
-			break;
-		}
-		pids = t;
-		sprintf(pids + len, "%d ", guest->task_pids[i]);
-		len += s;
-	}
-	if (pids) {
-		tracefs_instance_file_write(guest->instance, "set_event_pid", pids);
-		free(pids);
-	}
-	tracefs_instance_file_write(guest->instance, "events/kvm/kvm_entry/enable", "1");
-}
-
-static int map_vcpus(struct tep_event *event, struct tep_record *record,
-		     int cpu, void *context)
-{
-	struct trace_mapping *tmap = context;
-	unsigned long long val;
-	int type;
-	int pid;
-	int ret;
-	int i;
-
-	/* Do we have junk in the buffer? */
-	type = tep_data_type(event->tep, record);
-	if (type != tmap->kvm_entry->id)
-		return 0;
-
-	ret = tep_read_number_field(tmap->common_pid, record->data, &val);
-	if (ret < 0)
-		return 0;
-	pid = (int)val;
-
-	for (i = 0; tmap->pids[i] >= 0; i++) {
-		if (pid == tmap->pids[i])
-			break;
-	}
-	/* Is this thread one we care about ? */
-	if (tmap->pids[i] < 0)
-		return 0;
-
-	ret = tep_read_number_field(tmap->vcpu_id, record->data, &val);
-	if (ret < 0)
-		return 0;
-
-	cpu = (int)val;
-
-	/* Sanity check, warn? */
-	if (cpu >= tmap->max_cpus)
-		return 0;
-
-	/* Already have this one? Should we check if it is the same? */
-	if (tmap->map[cpu] >= 0)
-		return 0;
-
-	tmap->map[cpu] = pid;
-
-	/* Did we get them all */
-	for (i = 0; i < tmap->max_cpus; i++) {
-		if (tmap->map[i] < 0)
-			break;
-	}
-
-	return i == tmap->max_cpus;
-}
-
-static void stop_mapping_vcpus(int cpu_count, struct trace_guest *guest)
-{
-	struct trace_mapping tmap = { };
-	struct tep_handle *tep;
-	const char *systems[] = { "kvm", NULL };
-	int i;
-
-	if (!guest->instance)
-		return;
-
-	tmap.pids = guest->task_pids;
-	tmap.max_cpus = cpu_count;
-
-	tmap.map = malloc(sizeof(*tmap.map) * tmap.max_cpus);
-	if (!tmap.map)
-		return;
-
-	for (i = 0; i < tmap.max_cpus; i++)
-		tmap.map[i] = -1;
-
-	tracefs_instance_file_write(guest->instance, "events/kvm/kvm_entry/enable", "0");
-
-	tep = tracefs_local_events_system(NULL, systems);
-	if (!tep)
-		goto out;
-
-	tmap.kvm_entry = tep_find_event_by_name(tep, "kvm", "kvm_entry");
-	if (!tmap.kvm_entry)
-		goto out_free;
-
-	tmap.vcpu_id = tep_find_field(tmap.kvm_entry, "vcpu_id");
-	if (!tmap.vcpu_id)
-		goto out_free;
-
-	tmap.common_pid = tep_find_any_field(tmap.kvm_entry, "common_pid");
-	if (!tmap.common_pid)
-		goto out_free;
-
-	tracefs_iterate_raw_events(tep, guest->instance, NULL, 0, map_vcpus, &tmap);
-
-	for (i = 0; i < tmap.max_cpus; i++) {
-		if (tmap.map[i] < 0)
-			break;
-	}
-	/* We found all the mapped CPUs */
-	if (i == tmap.max_cpus) {
-		guest->cpu_pid = tmap.map;
-		guest->cpu_max = tmap.max_cpus;
-		tmap.map = NULL;
-	}
-
- out_free:
-	tep_free(tep);
- out:
-	free(tmap.map);
-	tracefs_instance_destroy(guest->instance);
-	tracefs_instance_free(guest->instance);
-}
-
-static struct tracecmd_time_sync *
-trace_tsync_as_host(int fd, unsigned long long trace_id,
-		    int loop_interval, int guest_id,
-		    int guest_cpus, const char *proto_name,
-		    const char *clock)
-{
-	struct tracecmd_time_sync *tsync;
-	struct trace_guest *guest;
-	int guest_pid = -1;
-
-	if (fd < 0)
-		return NULL;
-
-	if (guest_id >= 0) {
-		guest = trace_get_guest(guest_id, NULL);
-		if (guest == NULL)
-			return NULL;
-		guest_pid = guest->pid;
-		start_mapping_vcpus(guest);
-	}
-
-	tsync = tracecmd_tsync_with_guest(trace_id, loop_interval, fd,
-					  guest_pid, guest_cpus, proto_name,
-					  clock);
-
-	if (guest_id >= 0)
-		stop_mapping_vcpus(guest_cpus, guest);
-
-	return tsync;
-}
-
 static int host_tsync(struct common_record_context *ctx,
 		      struct buffer_instance *instance,
 		      unsigned int tsync_port, char *proto)
diff --git a/tracecmd/trace-tsync.c b/tracecmd/trace-tsync.c
new file mode 100644
index 000000000000..45ebb9c66a19
--- /dev/null
+++ b/tracecmd/trace-tsync.c
@@ -0,0 +1,242 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2021 Google Inc, Steven Rostedt <rostedt@goodmis.org>
+ * Copyright (C) 2020, VMware, Tzvetomir Stoyanov <tz.stoyanov@gmail.com>
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ */
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "trace-local.h"
+#include "trace-msg.h"
+
+struct trace_mapping {
+	struct tep_event		*kvm_entry;
+	struct tep_format_field		*vcpu_id;
+	struct tep_format_field		*common_pid;
+	int				*pids;
+	int				*map;
+	int				max_cpus;
+};
+
+static int map_vcpus(struct tep_event *event, struct tep_record *record,
+		     int cpu, void *context)
+{
+	struct trace_mapping *tmap = context;
+	unsigned long long val;
+	int type;
+	int pid;
+	int ret;
+	int i;
+
+	/* Do we have junk in the buffer? */
+	type = tep_data_type(event->tep, record);
+	if (type != tmap->kvm_entry->id)
+		return 0;
+
+	ret = tep_read_number_field(tmap->common_pid, record->data, &val);
+	if (ret < 0)
+		return 0;
+	pid = (int)val;
+
+	for (i = 0; tmap->pids[i] >= 0; i++) {
+		if (pid == tmap->pids[i])
+			break;
+	}
+	/* Is this thread one we care about ? */
+	if (tmap->pids[i] < 0)
+		return 0;
+
+	ret = tep_read_number_field(tmap->vcpu_id, record->data, &val);
+	if (ret < 0)
+		return 0;
+
+	cpu = (int)val;
+
+	/* Sanity check, warn? */
+	if (cpu >= tmap->max_cpus)
+		return 0;
+
+	/* Already have this one? Should we check if it is the same? */
+	if (tmap->map[cpu] >= 0)
+		return 0;
+
+	tmap->map[cpu] = pid;
+
+	/* Did we get them all */
+	for (i = 0; i < tmap->max_cpus; i++) {
+		if (tmap->map[i] < 0)
+			break;
+	}
+
+	return i == tmap->max_cpus;
+}
+
+static void start_mapping_vcpus(struct trace_guest *guest)
+{
+	char *pids = NULL;
+	char *t;
+	int len = 0;
+	int s;
+	int i;
+
+	if (!guest->task_pids)
+		return;
+
+	guest->instance = tracefs_instance_create("map_guest_pids");
+	if (!guest->instance)
+		return;
+
+	for (i = 0; guest->task_pids[i] >= 0; i++) {
+		s = snprintf(NULL, 0, "%d ", guest->task_pids[i]);
+		t = realloc(pids, len + s + 1);
+		if (!t) {
+			free(pids);
+			pids = NULL;
+			break;
+		}
+		pids = t;
+		sprintf(pids + len, "%d ", guest->task_pids[i]);
+		len += s;
+	}
+	if (pids) {
+		tracefs_instance_file_write(guest->instance, "set_event_pid", pids);
+		free(pids);
+	}
+	tracefs_instance_file_write(guest->instance, "events/kvm/kvm_entry/enable", "1");
+}
+
+static void stop_mapping_vcpus(int cpu_count, struct trace_guest *guest)
+{
+	struct trace_mapping tmap = { };
+	struct tep_handle *tep;
+	const char *systems[] = { "kvm", NULL };
+	int i;
+
+	if (!guest->instance)
+		return;
+
+	tmap.pids = guest->task_pids;
+	tmap.max_cpus = cpu_count;
+
+	tmap.map = malloc(sizeof(*tmap.map) * tmap.max_cpus);
+	if (!tmap.map)
+		return;
+
+	for (i = 0; i < tmap.max_cpus; i++)
+		tmap.map[i] = -1;
+
+	tracefs_instance_file_write(guest->instance, "events/kvm/kvm_entry/enable", "0");
+
+	tep = tracefs_local_events_system(NULL, systems);
+	if (!tep)
+		goto out;
+
+	tmap.kvm_entry = tep_find_event_by_name(tep, "kvm", "kvm_entry");
+	if (!tmap.kvm_entry)
+		goto out_free;
+
+	tmap.vcpu_id = tep_find_field(tmap.kvm_entry, "vcpu_id");
+	if (!tmap.vcpu_id)
+		goto out_free;
+
+	tmap.common_pid = tep_find_any_field(tmap.kvm_entry, "common_pid");
+	if (!tmap.common_pid)
+		goto out_free;
+
+	tracefs_iterate_raw_events(tep, guest->instance, NULL, 0, map_vcpus, &tmap);
+
+	for (i = 0; i < tmap.max_cpus; i++) {
+		if (tmap.map[i] < 0)
+			break;
+	}
+	/* We found all the mapped CPUs */
+	if (i == tmap.max_cpus) {
+		guest->cpu_pid = tmap.map;
+		guest->cpu_max = tmap.max_cpus;
+		tmap.map = NULL;
+	}
+
+ out_free:
+	tep_free(tep);
+ out:
+	free(tmap.map);
+	tracefs_instance_destroy(guest->instance);
+	tracefs_instance_free(guest->instance);
+}
+
+/**
+ * trace_tsync_as_host - tsync from the host side
+ * @fd: The descriptor to the peer for tsync
+ * @trace_id: The trace_id of the host
+ * @loop_interval: The loop interval for tsyncs that do periodic syncs
+ * @guest_id: The id for guests (negative if this is over network)
+ * @guest_cpus: The number of CPUs the guest has
+ * @proto_name: The protocol name to sync with
+ * @clock: The clock name to use for tracing
+ *
+ * Start the time synchronization from the host side.
+ * This will start the mapping of the virtual CPUs to host threads
+ * if it is a vsocket connection (not a network).
+ *
+ * Returns a pointer to the tsync descriptor on success or NULL on error.
+ */
+struct tracecmd_time_sync *
+trace_tsync_as_host(int fd, unsigned long long trace_id,
+		    int loop_interval, int guest_id,
+		    int guest_cpus, const char *proto_name,
+		    const char *clock)
+{
+	struct tracecmd_time_sync *tsync;
+	struct trace_guest *guest;
+	int guest_pid = -1;
+
+	if (fd < 0)
+		return NULL;
+
+	if (guest_id >= 0) {
+		guest = trace_get_guest(guest_id, NULL);
+		if (guest == NULL)
+			return NULL;
+		guest_pid = guest->pid;
+		start_mapping_vcpus(guest);
+	}
+
+	tsync = tracecmd_tsync_with_guest(trace_id, loop_interval, fd,
+					  guest_pid, guest_cpus, proto_name,
+					  clock);
+
+	if (guest_id >= 0)
+		stop_mapping_vcpus(guest_cpus, guest);
+
+	return tsync;
+}
+
+/**
+ * trace_tsync_a_guest - tsync from the guest side
+ * @fd: The file descriptor to the peer for tsync
+ * @tsync_proto: The protocol name to sync with
+ * @clock: The clock name to use for tracing
+ * @remote_id: The id to differentiate the remote server with
+ * @loca_id: The id to differentiate the local machine with
+ *
+ * Start the time synchronization from the guest side.
+ *
+ * Returns a pointer to the tsync descriptor on success or NULL on error.
+ */
+struct tracecmd_time_sync *
+trace_tsync_as_guest(int fd, const char *tsync_proto, const char *clock,
+	       unsigned int remote_id, unsigned int local_id)
+{
+	struct tracecmd_time_sync *tsync;
+
+	if (fd < 0)
+		return NULL;
+
+	tsync = tracecmd_tsync_with_host(fd, tsync_proto, clock,
+					 remote_id, local_id);
+	if (!tsync)
+		warning("Failed to negotiate timestamps synchronization with the host");
+
+	return tsync;
+}
-- 
2.35.1

[PATCH 17/26] trace-cmd msg: Add PROXY communication

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

Add communication for an agent proxy. When an agent is also running as a
proxy, it will be able to receive a TRACE_PROXY message. This will contain
the same data as a TRACE_REQ, but also include the number of CPUs the
recorder has (as the recorder may be running on the guest, and the agent
on the host), as well as the number of siblings guests that will be sent
afterward.

The sibling messages will come at a later time.

Currently there are no users of this API.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 .../include/private/trace-cmd-private.h       | 12 +++
 lib/trace-cmd/trace-msg.c                     | 90 ++++++++++++++++---
 2 files changed, 89 insertions(+), 13 deletions(-)

diff --git a/lib/trace-cmd/include/private/trace-cmd-private.h b/lib/trace-cmd/include/private/trace-cmd-private.h
index 77b387b494ee..8824dcae0b16 100644
--- a/lib/trace-cmd/include/private/trace-cmd-private.h
+++ b/lib/trace-cmd/include/private/trace-cmd-private.h
@@ -378,6 +378,7 @@ long tracecmd_flush_recording(struct tracecmd_recorder *recorder);
 enum tracecmd_msg_flags {
 	TRACECMD_MSG_FL_USE_TCP		= 1 << 0,
 	TRACECMD_MSG_FL_USE_VSOCK	= 1 << 1,
+	TRACECMD_MSG_FL_PROXY		= 1 << 2,
 };
 
 #define MSG_CACHE_FILE "/tmp/trace_msg_cacheXXXXXX"
@@ -431,10 +432,21 @@ int tracecmd_msg_send_trace_req(struct tracecmd_msg_handle *msg_handle,
 				int argc, char **argv, bool use_fifos,
 				unsigned long long trace_id,
 				struct tracecmd_tsync_protos *protos);
+int tracecmd_msg_send_trace_proxy(struct tracecmd_msg_handle *msg_handle,
+				  int argc, char **argv, bool use_fifos,
+				  unsigned long long trace_id,
+				  struct tracecmd_tsync_protos *protos,
+				  unsigned int nr_cpus,
+				  unsigned int siblings);
 int tracecmd_msg_recv_trace_req(struct tracecmd_msg_handle *msg_handle,
 				int *argc, char ***argv, bool *use_fifos,
 				unsigned long long *trace_id,
 				struct tracecmd_tsync_protos **protos);
+int tracecmd_msg_recv_trace_proxy(struct tracecmd_msg_handle *msg_handle,
+				  int *argc, char ***argv, bool *use_fifos,
+				  unsigned long long *trace_id,
+				  struct tracecmd_tsync_protos **protos,
+				  unsigned int *cpus, unsigned int *siblings);
 
 int tracecmd_msg_send_trace_resp(struct tracecmd_msg_handle *msg_handle,
 				 int nr_cpus, int page_size,
diff --git a/lib/trace-cmd/trace-msg.c b/lib/trace-cmd/trace-msg.c
index 9899fa8796e2..c16e8e81e50a 100644
--- a/lib/trace-cmd/trace-msg.c
+++ b/lib/trace-cmd/trace-msg.c
@@ -71,6 +71,12 @@ struct tracecmd_msg_trace_req {
 	u64 trace_id;
 } __packed;
 
+struct tracecmd_msg_trace_proxy {
+	struct tracecmd_msg_trace_req req;
+	be32 cpus;
+	be32 siblings;
+} __packed;
+
 struct tracecmd_msg_trace_resp {
 	be32 flags;
 	be32 cpus;
@@ -101,7 +107,8 @@ struct tracecmd_msg_header {
 	C(TRACE_REQ,	6,	sizeof(struct tracecmd_msg_trace_req)),	\
 	C(TRACE_RESP,	7,	sizeof(struct tracecmd_msg_trace_resp)),\
 	C(CLOSE_RESP,	8,	0),					\
-	C(TIME_SYNC,	9,	sizeof(struct tracecmd_msg_tsync)),
+	C(TIME_SYNC,	9,	sizeof(struct tracecmd_msg_tsync)),	\
+	C(TRACE_PROXY,	10,	sizeof(struct tracecmd_msg_trace_proxy)),
 
 #undef C
 #define C(a,b,c)	MSG_##a = b
@@ -134,6 +141,7 @@ struct tracecmd_msg {
 		struct tracecmd_msg_tinit	tinit;
 		struct tracecmd_msg_rinit	rinit;
 		struct tracecmd_msg_trace_req	trace_req;
+		struct tracecmd_msg_trace_proxy	trace_proxy;
 		struct tracecmd_msg_trace_resp	trace_resp;
 		struct tracecmd_msg_tsync	tsync;
 	};
@@ -1007,7 +1015,7 @@ static int make_trace_req(struct tracecmd_msg *msg, int argc, char **argv,
 	msg->buf = buf;
 	msg->hdr.size = htonl(ntohl(msg->hdr.size) + size);
 
-	return 0;
+	return size;
 }
 
 int tracecmd_msg_send_trace_req(struct tracecmd_msg_handle *msg_handle,
@@ -1026,6 +1034,26 @@ int tracecmd_msg_send_trace_req(struct tracecmd_msg_handle *msg_handle,
 	return tracecmd_msg_send(msg_handle, &msg);
 }
 
+int tracecmd_msg_send_trace_proxy(struct tracecmd_msg_handle *msg_handle,
+				  int argc, char **argv, bool use_fifos,
+				  unsigned long long trace_id,
+				  struct tracecmd_tsync_protos *protos,
+				  unsigned int nr_cpus,
+				  unsigned int siblings)
+{
+	struct tracecmd_msg msg;
+	int ret;
+
+	tracecmd_msg_init(MSG_TRACE_PROXY, &msg);
+	ret = make_trace_req(&msg, argc, argv, use_fifos, trace_id, protos);
+	if (ret < 0)
+		return ret;
+
+	msg.trace_proxy.cpus = htonl(nr_cpus);
+	msg.trace_proxy.siblings = htonl(siblings);
+	return tracecmd_msg_send(msg_handle, &msg);
+}
+
 static int get_trace_req_protos(char *buf, int length,
 				struct tracecmd_tsync_protos **protos)
 {
@@ -1119,16 +1147,12 @@ out:
 
 }
 
-/*
- * NOTE: On success, the returned `argv` should be freed with:
- *     free(argv[0]);
- *     free(argv);
- * and `tsync_protos` with free(tsync_protos);
- */
-int tracecmd_msg_recv_trace_req(struct tracecmd_msg_handle *msg_handle,
-				int *argc, char ***argv, bool *use_fifos,
-				unsigned long long *trace_id,
-				struct tracecmd_tsync_protos **protos)
+static int msg_recv_trace_req_proxy(struct tracecmd_msg_handle *msg_handle,
+				    int *argc, char ***argv, bool *use_fifos,
+				    unsigned long long *trace_id,
+				    struct tracecmd_tsync_protos **protos,
+				    unsigned int *cpus,
+				    unsigned int *siblings)
 {
 	struct tracecmd_msg msg;
 	unsigned int param_id;
@@ -1141,7 +1165,16 @@ int tracecmd_msg_recv_trace_req(struct tracecmd_msg_handle *msg_handle,
 	if (ret < 0)
 		return ret;
 
-	if (ntohl(msg.hdr.cmd) != MSG_TRACE_REQ) {
+	switch (ntohl(msg.hdr.cmd)) {
+	case MSG_TRACE_PROXY:
+		if (cpus)
+			*cpus = ntohl(msg.trace_proxy.cpus);
+		if (siblings)
+			*siblings = ntohl(msg.trace_proxy.siblings);
+		/* fall through */
+	case MSG_TRACE_REQ:
+		break;
+	default:
 		ret = -ENOTSUP;
 		goto out;
 	}
@@ -1192,6 +1225,37 @@ out:
 	return ret;
 }
 
+/*
+ * NOTE: On success, the returned `argv` should be freed with:
+ *     free(argv[0]);
+ *     free(argv);
+ * and `tsync_protos` with free(tsync_protos);
+ */
+int tracecmd_msg_recv_trace_req(struct tracecmd_msg_handle *msg_handle,
+				int *argc, char ***argv, bool *use_fifos,
+				unsigned long long *trace_id,
+				struct tracecmd_tsync_protos **protos)
+{
+	return msg_recv_trace_req_proxy(msg_handle, argc, argv, use_fifos,
+					trace_id, protos, NULL, NULL);
+}
+
+/*
+ * NOTE: On success, the returned `argv` should be freed with:
+ *     free(argv[0]);
+ *     free(argv);
+ * and `tsync_protos` with free(tsync_protos);
+ */
+int tracecmd_msg_recv_trace_proxy(struct tracecmd_msg_handle *msg_handle,
+				  int *argc, char ***argv, bool *use_fifos,
+				  unsigned long long *trace_id,
+				  struct tracecmd_tsync_protos **protos,
+				  unsigned int *cpus, unsigned int *siblings)
+{
+	return msg_recv_trace_req_proxy(msg_handle, argc, argv, use_fifos,
+					trace_id, protos, cpus, siblings);
+}
+
 /**
  * tracecmd_msg_send_time_sync - Send a time sync packet
  * @msg_handle: message handle, holding the communication context
-- 
2.35.1

[PATCH 18/26] trace-cmd: Add agent proxy communications between record and agent

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

Add:

  trace-cmd agent -P <cid>

to allow an agent to also act as a proxy server, where it can be run on a
host and connect with a guest. The <cid> is the guest's cid that it will
connect to (it will only connect to a specified guest).

Add:

  trace-cmd record --proxy ...

That acts the same as -A (for an agent) but will send the proxy connection
to the agent. It is expected to run on a privilege guest that the host is
aware of (as denoted by the <cid> in the -P option for the agent).

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/include/trace-local.h |  2 ++
 tracecmd/trace-agent.c         | 56 ++++++++++++++++++++++++++++------
 tracecmd/trace-record.c        | 38 ++++++++++++++++++-----
 3 files changed, 78 insertions(+), 18 deletions(-)

diff --git a/tracecmd/include/trace-local.h b/tracecmd/include/trace-local.h
index 5febe9a60a54..92d005c7b12c 100644
--- a/tracecmd/include/trace-local.h
+++ b/tracecmd/include/trace-local.h
@@ -190,6 +190,7 @@ enum buffer_instance_flags {
 	BUFFER_FL_HAS_CLOCK	= 1 << 4,
 	BUFFER_FL_TSC2NSEC	= 1 << 5,
 	BUFFER_FL_NETWORK	= 1 << 6,
+	BUFFER_FL_PROXY		= 1 << 7,
 };
 
 struct func_list {
@@ -305,6 +306,7 @@ extern struct buffer_instance *first_instance;
 
 #define is_agent(instance)	((instance)->flags & BUFFER_FL_AGENT)
 #define is_guest(instance)	((instance)->flags & BUFFER_FL_GUEST)
+#define is_proxy(instance)	((instance)->flags & BUFFER_FL_PROXY)
 #define is_network(instance)	((instance)->flags & BUFFER_FL_NETWORK)
 
 #define START_PORT_SEARCH 1500
diff --git a/tracecmd/trace-agent.c b/tracecmd/trace-agent.c
index 920094702b6d..360079297b88 100644
--- a/tracecmd/trace-agent.c
+++ b/tracecmd/trace-agent.c
@@ -144,17 +144,22 @@ static int wait_for_connection(int fd)
 	return sd;
 }
 
-static void agent_handle(int sd, int nr_cpus, int page_size, const char *network)
+static void agent_handle(int sd, int nr_cpus, int page_size,
+			 int cid, int rcid, const char *network)
 {
 	struct tracecmd_tsync_protos *tsync_protos = NULL;
 	struct tracecmd_time_sync *tsync = NULL;
 	struct tracecmd_msg_handle *msg_handle;
 	const char *tsync_proto = NULL;
+	unsigned long long peer_trace_id;
 	unsigned long long trace_id;
+	unsigned long flags = rcid >= 0 ? TRACECMD_MSG_FL_PROXY : 0;
 	unsigned int remote_id;
 	unsigned int local_id;
 	unsigned int tsync_port = 0;
 	unsigned int *ports;
+	unsigned int client_cpus = 0;
+	unsigned int guests = 0;
 	char **argv = NULL;
 	int argc = 0;
 	bool use_fifos;
@@ -167,13 +172,20 @@ static void agent_handle(int sd, int nr_cpus, int page_size, const char *network
 	if (!fds || !ports)
 		die("Failed to allocate memory");
 
-	msg_handle = tracecmd_msg_handle_alloc(sd, 0);
+	msg_handle = tracecmd_msg_handle_alloc(sd, flags);
 	if (!msg_handle)
 		die("Failed to allocate message handle");
 
-	ret = tracecmd_msg_recv_trace_req(msg_handle, &argc, &argv,
-					  &use_fifos, &trace_id,
-					  &tsync_protos);
+	if (rcid >= 0)
+		ret = tracecmd_msg_recv_trace_proxy(msg_handle, &argc, &argv,
+						    &use_fifos, &peer_trace_id,
+						    &tsync_protos,
+						    &client_cpus,
+						    &guests);
+	else
+		ret = tracecmd_msg_recv_trace_req(msg_handle, &argc, &argv,
+						  &use_fifos, &peer_trace_id,
+						  &tsync_protos);
 	if (ret < 0)
 		die("Failed to receive trace request");
 
@@ -270,14 +282,15 @@ static pid_t do_fork()
 	return fork();
 }
 
-static void agent_serve(unsigned int port, bool do_daemon, const char *network)
+static void agent_serve(unsigned int port, bool do_daemon, int proxy_id,
+			const char *network)
 {
 	struct sockaddr_storage net_addr;
 	struct sockaddr *addr = NULL;
 	socklen_t *addr_len_p = NULL;
 	socklen_t addr_len = sizeof(net_addr);
 	int sd, cd, nr_cpus;
-	unsigned int cid;
+	unsigned int cid = -1, rcid = -1;
 	pid_t pid;
 
 	signal(SIGCHLD, handle_sigchld);
@@ -316,6 +329,21 @@ static void agent_serve(unsigned int port, bool do_daemon, const char *network)
 				continue;
 			die("accept");
 		}
+		if (proxy_id >= 0) {
+			/* Only works with vsockets */
+			if (get_vsocket_params(cd, NULL, &rcid) < 0) {
+				dprint("Failed to find connected cid");
+				close(cd);
+				continue;
+			}
+			if (rcid != proxy_id) {
+				dprint("Cid %d does not match expected cid %d\n",
+				       rcid, proxy_id);
+				close(cd);
+				continue;
+			}
+		}
+
 		if (tracecmd_get_debug())
 			trace_print_connection(cd, network);
 
@@ -332,7 +360,7 @@ static void agent_serve(unsigned int port, bool do_daemon, const char *network)
 		if (pid == 0) {
 			close(sd);
 			signal(SIGCHLD, SIG_DFL);
-			agent_handle(cd, nr_cpus, page_size, network);
+			agent_handle(cd, nr_cpus, page_size, cid, rcid, network);
 		}
 		if (pid > 0)
 			handler_pid = pid;
@@ -352,6 +380,7 @@ void trace_agent(int argc, char **argv)
 	bool do_daemon = false;
 	unsigned int port = TRACE_AGENT_DEFAULT_PORT;
 	const char *network = NULL;
+	int proxy_id = -1;
 
 	if (argc < 2)
 		usage(argv);
@@ -369,7 +398,7 @@ void trace_agent(int argc, char **argv)
 			{NULL, 0, NULL, 0}
 		};
 
-		c = getopt_long(argc-1, argv+1, "+hp:DN:",
+		c = getopt_long(argc-1, argv+1, "+hp:DN:P:",
 				long_options, &option_index);
 		if (c == -1)
 			break;
@@ -382,10 +411,17 @@ void trace_agent(int argc, char **argv)
 			break;
 		case 'p':
 			port = atoi(optarg);
+			if (proxy_id >= 0)
+				die("-N cannot be used with -P");
 			break;
 		case 'D':
 			do_daemon = true;
 			break;
+		case 'P':
+			proxy_id = atoi(optarg);
+			if (network)
+				die("-P cannot be used with -N");
+			break;
 		case DO_DEBUG:
 			tracecmd_set_debug(true);
 			break;
@@ -401,5 +437,5 @@ void trace_agent(int argc, char **argv)
 	if (optind < argc-1)
 		usage(argv);
 
-	agent_serve(port, do_daemon, network);
+	agent_serve(port, do_daemon, proxy_id, network);
 }
diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index bf85e7936b73..7ffea4c7c76c 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -3862,6 +3862,7 @@ static void connect_to_agent(struct common_record_context *ctx,
 	unsigned int *ports;
 	int i, *fds = NULL;
 	bool use_fifos = false;
+	int siblings = 0;
 
 	if (!no_fifos) {
 		nr_fifos = open_guest_fifos(instance->name, &fds);
@@ -3892,11 +3893,19 @@ static void connect_to_agent(struct common_record_context *ctx,
 	if (instance->tsync_loop_interval >= 0)
 		tracecmd_tsync_proto_getall(&protos, instance->clock, role);
 
-	ret = tracecmd_msg_send_trace_req(msg_handle, instance->argc,
-					  instance->argv, use_fifos,
-					  top_instance.trace_id, protos);
+	if (is_proxy(instance))
+		ret = tracecmd_msg_send_trace_proxy(msg_handle, instance->argc,
+						    instance->argv, use_fifos,
+						    top_instance.trace_id, protos,
+						    tracecmd_count_cpus(),
+						    siblings);
+	else
+		ret = tracecmd_msg_send_trace_req(msg_handle, instance->argc,
+						  instance->argv, use_fifos,
+						  top_instance.trace_id, protos);
 	if (ret < 0)
-		die("Failed to send trace request");
+		die("Failed to send trace %s",
+		    is_proxy(instance) ? "proxy" : "request");
 
 	if (protos) {
 		free(protos->names);
@@ -5623,6 +5632,7 @@ enum {
 	OPT_cmdlines_size	= 258,
 	OPT_poll		= 259,
 	OPT_name		= 260,
+	OPT_proxy		= 261
 };
 
 void trace_stop(int argc, char **argv)
@@ -6005,6 +6015,7 @@ static void parse_record_options(int argc,
 	char *sav;
 	int name_counter = 0;
 	int negative = 0;
+	bool is_proxy = false;
 	struct buffer_instance *instance, *del_list = NULL;
 	int do_children = 0;
 	int fpids_count = 0;
@@ -6045,6 +6056,7 @@ static void parse_record_options(int argc,
 			{"verbose", optional_argument, NULL, OPT_verbose},
 			{"compression", required_argument, NULL, OPT_compression},
 			{"file-version", required_argument, NULL, OPT_file_ver},
+			{"proxy", required_argument, NULL, OPT_proxy},
 			{NULL, 0, NULL, 0}
 		};
 
@@ -6060,7 +6072,8 @@ static void parse_record_options(int argc,
 		 * If the current instance is to record a guest, then save
 		 * all the arguments for this instance.
 		 */
-		if (c != 'B' && c != 'A' && c != OPT_name && is_guest(ctx->instance)) {
+		if (c != 'B' && (c != 'A' || is_proxy) && c != OPT_name &&
+		    is_guest(ctx->instance)) {
 			add_arg(ctx->instance, c, opts, long_options, optarg);
 			if (c == 'C')
 				ctx->instance->flags |= BUFFER_FL_HAS_CLOCK;
@@ -6132,12 +6145,16 @@ static void parse_record_options(int argc,
 				die("Failed to allocate name");
 			break;
 
+		case OPT_proxy:
+			is_proxy = true;
+			/* fall through */
 		case 'A': {
 			char *name = NULL;
 			int cid = -1, port = -1;
 
 			if (!IS_RECORD(ctx))
-				die("-A is only allowed for record operations");
+				die("%s is only allowed for record operations",
+				    is_proxy ? "--proxy" : "-A");
 
 			name = parse_guest_name(optarg, &cid, &port, &result);
 			if (cid == -1 && !result)
@@ -6164,6 +6181,9 @@ static void parse_record_options(int argc,
 				ctx->instance->port_type = USE_TCP;
 			}
 
+			if (is_proxy)
+				ctx->instance->flags |= BUFFER_FL_PROXY;
+
 			ctx->instance->flags |= BUFFER_FL_GUEST;
 			ctx->instance->result = result;
 			ctx->instance->cid = cid;
@@ -6386,6 +6406,8 @@ static void parse_record_options(int argc,
 			ctx->instance->buffer_size = atoi(optarg);
 			break;
 		case 'B':
+			/* Turn off proxy for the next options */
+			is_proxy = false;
 			ctx->instance = allocate_instance(optarg);
 			if (!ctx->instance)
 				die("Failed to create instance");
@@ -7033,6 +7055,7 @@ void trace_record(int argc, char **argv)
  * @argv: The arguments to pass to the record session
  * @use_fifos: True if fifos are used instead of sockets.
  * @trace_id: The agent's trace_id
+ * @rcid: Remote cid if the agent is a proxy, negative otherwise.
  * @host: Set if this is an IP connection and not a vsocket one
  *
  * This is used to enable tracing via the record command just
@@ -7044,8 +7067,7 @@ void trace_record(int argc, char **argv)
  */
 int trace_record_agent(struct tracecmd_msg_handle *msg_handle,
 		       int cpus, int *fds,
-		       int argc, char **argv,
-		       bool use_fifos,
+		       int argc, char **argv, bool use_fifos,
 		       unsigned long long trace_id, const char *host)
 {
 	struct common_record_context ctx;
-- 
2.35.1

[PATCH 19/26] trace-cmd msg: Keep track of offset of flushed cache

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

The proxy agent will need to send data in two chunks, and will need to
save the offset of the flushed cache so that it can be used later.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 .../include/private/trace-cmd-private.h       |  1 +
 lib/trace-cmd/trace-msg.c                     | 29 +++++++++++++++----
 2 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/lib/trace-cmd/include/private/trace-cmd-private.h b/lib/trace-cmd/include/private/trace-cmd-private.h
index 8824dcae0b16..ef2351e8491f 100644
--- a/lib/trace-cmd/include/private/trace-cmd-private.h
+++ b/lib/trace-cmd/include/private/trace-cmd-private.h
@@ -389,6 +389,7 @@ struct tracecmd_msg_handle {
 	short			cpu_count;
 	short			version;	/* Current protocol version */
 	unsigned long		flags;
+	off64_t			cache_start_offset;
 	bool			done;
 	bool			cache;
 	int			cfd;
diff --git a/lib/trace-cmd/trace-msg.c b/lib/trace-cmd/trace-msg.c
index c16e8e81e50a..963ef94e977b 100644
--- a/lib/trace-cmd/trace-msg.c
+++ b/lib/trace-cmd/trace-msg.c
@@ -184,13 +184,27 @@ static int __msg_write(int fd, struct tracecmd_msg *msg, bool network)
 
 __hidden off64_t msg_lseek(struct tracecmd_msg_handle *msg_handle, off64_t offset, int whence)
 {
+	off64_t cache_offset = msg_handle->cache_start_offset;
+	off64_t ret;
+
 	/*
 	 * lseek works only if the handle is in cache mode,
 	 * cannot seek on a network socket
 	 */
 	if (!msg_handle->cache || msg_handle->cfd < 0)
 		return (off64_t)-1;
-	return lseek64(msg_handle->cfd, offset, whence);
+
+	if (whence == SEEK_SET) {
+		if (offset < cache_offset)
+			return (off64_t)-1;
+		offset -= cache_offset;
+	}
+
+	ret = lseek64(msg_handle->cfd, offset, whence);
+	if (ret == (off64_t)-1)
+		return ret;
+
+	return ret + cache_offset;
 }
 
 static int msg_write(struct tracecmd_msg_handle *msg_handle, struct tracecmd_msg *msg)
@@ -618,15 +632,16 @@ int tracecmd_msg_handle_cache(struct tracecmd_msg_handle *msg_handle)
 static int flush_cache(struct tracecmd_msg_handle *msg_handle)
 {
 	char buf[MSG_MAX_DATA_LEN];
+	int fd = msg_handle->cfd;
 	int ret;
 
-	if (!msg_handle->cache || msg_handle->cfd < 0)
+	if (!msg_handle->cache || fd < 0)
 		return 0;
 	msg_handle->cache = false;
-	if (lseek64(msg_handle->cfd, 0, SEEK_SET) == (off64_t)-1)
+	if (lseek64(fd, 0, SEEK_SET) == (off64_t)-1)
 		return -1;
 	do {
-		ret = read(msg_handle->cfd, buf, MSG_MAX_DATA_LEN);
+		ret = read(fd, buf, MSG_MAX_DATA_LEN);
 		if (ret <= 0)
 			break;
 		ret = tracecmd_msg_data_send(msg_handle, buf, ret);
@@ -634,7 +649,11 @@ static int flush_cache(struct tracecmd_msg_handle *msg_handle)
 			break;
 	} while (ret >= 0);
 
-	close(msg_handle->cfd);
+	msg_handle->cache_start_offset = lseek64(fd, 0, SEEK_CUR);
+	if (msg_handle->cache_start_offset == (off64_t)-1)
+		return -1;
+
+	close(fd);
 	msg_handle->cfd = -1;
 	return ret;
 }
-- 
2.35.1

[PATCH 20/26] trace-cmd library: Add tracecmd_prepare_options()

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

Add a new interface tracecmd_prepare_options() that can be used by
tracecmd_msg to update the next options before flushing over the network.
Once the last options is written to the network, it can not be updated to
point to the next options section.

The tracecmd_prepare_options() will update the last options in the file to
point to the offset of the next one (before they are written). The
location of the next options must be known (obviously), but this means
that the data can be flushed over the network before the next options are
written. The use case is where there's options that need to be added when
the trace is done, but the beginning of the trace file is already sent out
over the network.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 .../include/private/trace-cmd-private.h       |  1 +
 lib/trace-cmd/trace-output.c                  | 90 +++++++++++++++++--
 2 files changed, 85 insertions(+), 6 deletions(-)

diff --git a/lib/trace-cmd/include/private/trace-cmd-private.h b/lib/trace-cmd/include/private/trace-cmd-private.h
index ef2351e8491f..28fa712c4de8 100644
--- a/lib/trace-cmd/include/private/trace-cmd-private.h
+++ b/lib/trace-cmd/include/private/trace-cmd-private.h
@@ -333,6 +333,7 @@ int tracecmd_write_buffer_info(struct tracecmd_output *handle);
 
 int tracecmd_write_cpus(struct tracecmd_output *handle, int cpus);
 int tracecmd_write_cmdlines(struct tracecmd_output *handle);
+int tracecmd_prepare_options(struct tracecmd_output *handle, off64_t offset, int whence);
 int tracecmd_write_options(struct tracecmd_output *handle);
 int tracecmd_write_meta_strings(struct tracecmd_output *handle);
 int tracecmd_append_options(struct tracecmd_output *handle);
diff --git a/lib/trace-cmd/trace-output.c b/lib/trace-cmd/trace-output.c
index ca7132e113c0..1fc41424eca7 100644
--- a/lib/trace-cmd/trace-output.c
+++ b/lib/trace-cmd/trace-output.c
@@ -69,6 +69,7 @@ struct tracecmd_output {
 	unsigned long		strings_offs;
 
 	unsigned long long	options_start;
+	unsigned long long	options_next;
 	bool			big_endian;
 	bool			do_compress;
 	struct tracecmd_compression *compress;
@@ -1841,6 +1842,66 @@ static int write_options_v6(struct tracecmd_output *handle)
 	return 0;
 }
 
+static int update_options_start(struct tracecmd_output *handle, off64_t offset)
+{
+	if (do_lseek(handle, handle->options_start, SEEK_SET) == (off64_t)-1)
+		return -1;
+	offset = convert_endian_8(handle, offset);
+	if (do_write_check(handle, &offset, 8))
+		return -1;
+	return 0;
+}
+
+/**
+ * tracecmd_pepare_options - perpare a previous options for the next
+ * @handle: The handle to update the options for.
+ * @offset: The offset to set the previous options to.
+ * @whence: Where in the file to offset from.
+ *
+ * In a case of cached writes for network access, the options offset
+ * cannot be written once it goes over the network. This is used
+ * to update the next options to a known location.
+ *
+ * tracecmd_write_options() must be called when the offset is at the next
+ * location, otherwise the data file will end up corrupted.
+ *
+ * Returns zero on success and -1 on error.
+ */
+int tracecmd_prepare_options(struct tracecmd_output *handle, off64_t offset, int whence)
+{
+	tsize_t curr;
+	int ret;
+
+	/* No options to start with? */
+	if (!handle->options_start)
+		return 0;
+
+	curr = do_lseek(handle, 0, SEEK_CUR);
+
+	switch (whence) {
+	case SEEK_SET:
+		/* just use offset */
+		break;
+	case SEEK_CUR:
+		offset += curr;
+		break;
+	case SEEK_END:
+		offset = do_lseek(handle, offset, SEEK_END);
+		if (offset == (off64_t)-1)
+			return -1;
+		break;
+	}
+	ret = update_options_start(handle, offset);
+	if (ret < 0)
+		return -1;
+
+	handle->options_next = offset;
+
+	curr = do_lseek(handle, curr, SEEK_SET);
+
+	return curr == -1 ? -1 : 0;
+}
+
 static int write_options(struct tracecmd_output *handle)
 {
 	struct tracecmd_option *options;
@@ -1849,6 +1910,7 @@ static int write_options(struct tracecmd_output *handle)
 	unsigned int endian4;
 	bool new = false;
 	tsize_t offset;
+	int ret;
 
 	/* Check if there are unsaved options */
 	list_for_each_entry(options, &handle->options, list) {
@@ -1857,18 +1919,34 @@ static int write_options(struct tracecmd_output *handle)
 			break;
 		}
 	}
-	if (!new)
+	/*
+	 * Even if there are no new options, if options_next is set, it requires
+	 * adding a new empty options section as the previous one already
+	 * points to it.
+	 */
+	if (!new && !handle->options_next)
 		return 0;
 	offset = do_lseek(handle, 0, SEEK_CUR);
 
+	if (handle->options_next) {
+		/* options_start was already updated */
+		if (handle->options_next != offset) {
+			tracecmd_warning("Options offset (%lld) does not match expected (%lld)",
+					 offset, handle->options_next);
+			return -1;
+		}
+		handle->options_next = 0;
+		/* Will be updated at the end */
+		handle->options_start = 0;
+	}
+
 	/* Append to the previous options section, if any */
 	if (handle->options_start) {
-		if (do_lseek(handle, handle->options_start, SEEK_SET) == (off64_t)-1)
-			return -1;
-		endian8 = convert_endian_8(handle, offset);
-		if (do_write_check(handle, &endian8, 8))
+		ret = update_options_start(handle, offset);
+		if (ret < 0)
 			return -1;
-		if (do_lseek(handle, offset, SEEK_SET) == (off_t)-1)
+		offset = do_lseek(handle, offset, SEEK_SET);
+		if (offset == (off_t)-1)
 			return -1;
 	}
 
-- 
2.35.1

[PATCH 21/26] trace-cmd library: Add tracecmd_msg_flush_data()

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

Add tracecmd_msg_flush_data() that will flush the current cached data to
the network and restart the cache file.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 .../include/private/trace-cmd-private.h       |  1 +
 lib/trace-cmd/trace-msg.c                     | 20 +++++++++++++++++++
 2 files changed, 21 insertions(+)

diff --git a/lib/trace-cmd/include/private/trace-cmd-private.h b/lib/trace-cmd/include/private/trace-cmd-private.h
index 28fa712c4de8..053866d547af 100644
--- a/lib/trace-cmd/include/private/trace-cmd-private.h
+++ b/lib/trace-cmd/include/private/trace-cmd-private.h
@@ -416,6 +416,7 @@ int tracecmd_msg_send_init_data(struct tracecmd_msg_handle *msg_handle,
 int tracecmd_msg_data_send(struct tracecmd_msg_handle *msg_handle,
 			       const char *buf, int size);
 int tracecmd_msg_finish_sending_data(struct tracecmd_msg_handle *msg_handle);
+int tracecmd_msg_flush_data(struct tracecmd_msg_handle *msg_handle);
 int tracecmd_msg_send_close_msg(struct tracecmd_msg_handle *msg_handle);
 int tracecmd_msg_send_close_resp_msg(struct tracecmd_msg_handle *msg_handle);
 int tracecmd_msg_wait_close(struct tracecmd_msg_handle *msg_handle);
diff --git a/lib/trace-cmd/trace-msg.c b/lib/trace-cmd/trace-msg.c
index 963ef94e977b..a0c7f1818b0f 100644
--- a/lib/trace-cmd/trace-msg.c
+++ b/lib/trace-cmd/trace-msg.c
@@ -823,6 +823,26 @@ int tracecmd_msg_data_send(struct tracecmd_msg_handle *msg_handle,
 	return ret;
 }
 
+/**
+ * tracecmd_msg_flush_data - Send the current cache data over the network
+ * @msg_handle: message handle, holding the communication context
+ *
+ * Send the content in the cache file over the nework, reset the file
+ * and start the cache up again (with nothing in it).
+ */
+int tracecmd_msg_flush_data(struct tracecmd_msg_handle *msg_handle)
+{
+	struct tracecmd_msg msg;
+	int ret;
+
+	flush_cache(msg_handle);
+	tracecmd_msg_init(MSG_FIN_DATA, &msg);
+	ret = tracecmd_msg_send(msg_handle, &msg);
+	if (ret < 0)
+		return ret;
+	return tracecmd_msg_handle_cache(msg_handle);
+}
+
 int tracecmd_msg_finish_sending_data(struct tracecmd_msg_handle *msg_handle)
 {
 	struct tracecmd_msg msg;
-- 
2.35.1

[PATCH 22/26] trace-cmd agent proxy: Allow agent to send more meta data after trace

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

The options of the agent proxy trace file will need to include timings
and guest data that is added after the trace has completed. Do not close
the network handle for the meta data before the trace. Keep it open for
the proxy agent and finish writing the options at the end of the trace.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/include/trace-local.h |  5 +++++
 tracecmd/trace-record.c        | 39 +++++++++++++++++++++++++++++-----
 2 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/tracecmd/include/trace-local.h b/tracecmd/include/trace-local.h
index 92d005c7b12c..295be5dda7b5 100644
--- a/tracecmd/include/trace-local.h
+++ b/tracecmd/include/trace-local.h
@@ -280,6 +280,8 @@ struct buffer_instance {
 	int			buffer_size;
 	int			cpu_count;
 
+	int			proxy_fd;
+
 	int			argc;
 	char			**argv;
 
@@ -308,6 +310,9 @@ extern struct buffer_instance *first_instance;
 #define is_guest(instance)	((instance)->flags & BUFFER_FL_GUEST)
 #define is_proxy(instance)	((instance)->flags & BUFFER_FL_PROXY)
 #define is_network(instance)	((instance)->flags & BUFFER_FL_NETWORK)
+#define is_proxy_server(instance)					\
+	((instance)->msg_handle &&					\
+	 (instance)->msg_handle->flags & TRACECMD_MSG_FL_PROXY)
 
 #define START_PORT_SEARCH 1500
 #define MAX_PORT_SEARCH 6000
diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index 7ffea4c7c76c..a6829565a679 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -720,8 +720,15 @@ static void tell_guests_to_stop(struct common_record_context *ctx)
 
 	/* Send close message to guests */
 	for_all_instances(instance) {
-		if (is_guest(instance))
+		if (is_guest(instance)) {
 			tracecmd_msg_send_close_msg(instance->msg_handle);
+			if (is_proxy(instance) && instance->proxy_fd >= 0) {
+				/* The proxy will send more data now */
+				if (tracecmd_msg_read_data(instance->msg_handle, instance->proxy_fd))
+					warning("Failed receiving finishing metadata");
+				close(instance->proxy_fd);
+			}
+		}
 	}
 
 	for_all_instances(instance) {
@@ -3974,7 +3981,15 @@ static void setup_guest(struct buffer_instance *instance)
 	/* Start reading tracing metadata */
 	if (tracecmd_msg_read_data(msg_handle, fd))
 		die("Failed receiving metadata");
-	close(fd);
+
+	/*
+	 * If connected to a proxy, then it still needs to send
+	 * the host / guest timings from its POV.
+	 */
+	if (is_proxy(instance))
+		instance->proxy_fd = fd;
+	else
+		close(fd);
 }
 
 static void setup_agent(struct buffer_instance *instance,
@@ -3987,9 +4002,14 @@ static void setup_agent(struct buffer_instance *instance,
 	tracecmd_write_cmdlines(network_handle);
 	tracecmd_write_cpus(network_handle, instance->cpu_count);
 	tracecmd_write_buffer_info(network_handle);
-	tracecmd_write_options(network_handle);
-	tracecmd_write_meta_strings(network_handle);
-	tracecmd_msg_finish_sending_data(instance->msg_handle);
+	if (instance->msg_handle->flags & TRACECMD_MSG_FL_PROXY) {
+		tracecmd_prepare_options(network_handle, 0, SEEK_CUR);
+		tracecmd_msg_flush_data(instance->msg_handle);
+	} else {
+		tracecmd_write_options(network_handle);
+		tracecmd_write_meta_strings(network_handle);
+		tracecmd_msg_finish_sending_data(instance->msg_handle);
+	}
 	instance->network_handle = network_handle;
 }
 
@@ -4016,6 +4036,9 @@ static void start_threads(enum trace_type type, struct common_record_context *ct
 		int *brass = NULL;
 		int x, pid;
 
+		/* May be set by setup_guest() but all others is -1 */
+		instance->proxy_fd = -1;
+
 		if (is_agent(instance)) {
 			setup_agent(instance, ctx);
 		} else if (is_guest(instance)) {
@@ -6892,6 +6915,12 @@ static void record_trace(int argc, char **argv,
 	if (!latency)
 		wait_threads();
 
+	if (is_proxy_server(ctx->instance) && ctx->instance->network_handle) {
+		tracecmd_write_options(ctx->instance->network_handle);
+		tracecmd_write_meta_strings(ctx->instance->network_handle);
+		tracecmd_msg_finish_sending_data(ctx->instance->msg_handle);
+	}
+
 	if (IS_RECORD(ctx)) {
 		record_data(ctx);
 		delete_thread_data();
-- 
2.35.1

[PATCH 23/26] trace-cmd agent proxy: Add the remote guest cid to guest list

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

When the trace-cmd agent is passed the proxy flag "-P" with the guest
CID that it can talk to, add it to the list of guests, as it should be
able to find all the host tasks that control the guest.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/trace-agent.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tracecmd/trace-agent.c b/tracecmd/trace-agent.c
index 360079297b88..4430cb5506ff 100644
--- a/tracecmd/trace-agent.c
+++ b/tracecmd/trace-agent.c
@@ -22,6 +22,8 @@
 #include "trace-local.h"
 #include "trace-msg.h"
 
+#define GUEST_NAME	"::GUEST::"
+
 #define dprint(fmt, ...)	tracecmd_debug(fmt, ##__VA_ARGS__)
 
 static void make_vsocks(int nr, int *fds, unsigned int *ports)
@@ -377,6 +379,7 @@ enum {
 
 void trace_agent(int argc, char **argv)
 {
+	struct trace_guest *guest;
 	bool do_daemon = false;
 	unsigned int port = TRACE_AGENT_DEFAULT_PORT;
 	const char *network = NULL;
@@ -419,8 +422,11 @@ void trace_agent(int argc, char **argv)
 			break;
 		case 'P':
 			proxy_id = atoi(optarg);
-			if (network)
-				die("-P cannot be used with -N");
+
+			guest = trace_get_guest(proxy_id, GUEST_NAME);
+			if (!guest)
+				die("Failed to allocate guest instance");
+
 			break;
 		case DO_DEBUG:
 			tracecmd_set_debug(true);
-- 
2.35.1

[PATCH 24/26] trace-cmd agent-proxy: Send options at the end of the trace

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

The host controls the time shift for the guest, and when the host is
running as an agent proxy, it must be able to send that data to the
guest to be saved it the guests file. But the guest running the recorder
does not create its file until the end of the trace.

Send options for the time shift at the end of the trace (will later add
the time synchronization too). The guest recorder will then add this to
its trace.dat file.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 .../include/private/trace-cmd-private.h       |   6 +
 lib/trace-cmd/include/trace-cmd-local.h       |   2 +
 lib/trace-cmd/trace-msg.c                     | 148 ++++++++++++---
 lib/trace-cmd/trace-output.c                  | 177 ++++++++++++++----
 tracecmd/trace-record.c                       |  41 +++-
 5 files changed, 317 insertions(+), 57 deletions(-)

diff --git a/lib/trace-cmd/include/private/trace-cmd-private.h b/lib/trace-cmd/include/private/trace-cmd-private.h
index 053866d547af..5606e132cba1 100644
--- a/lib/trace-cmd/include/private/trace-cmd-private.h
+++ b/lib/trace-cmd/include/private/trace-cmd-private.h
@@ -421,6 +421,8 @@ int tracecmd_msg_send_close_msg(struct tracecmd_msg_handle *msg_handle);
 int tracecmd_msg_send_close_resp_msg(struct tracecmd_msg_handle *msg_handle);
 int tracecmd_msg_wait_close(struct tracecmd_msg_handle *msg_handle);
 int tracecmd_msg_wait_close_resp(struct tracecmd_msg_handle *msg_handle);
+int tracecmd_msg_cont(struct tracecmd_msg_handle *msg_handle);
+int tracecmd_msg_wait(struct tracecmd_msg_handle *msg_handle);
 
 /* for server */
 int tracecmd_msg_initial_setting(struct tracecmd_msg_handle *msg_handle);
@@ -430,6 +432,10 @@ int tracecmd_msg_read_data(struct tracecmd_msg_handle *msg_handle, int ofd);
 int tracecmd_msg_collect_data(struct tracecmd_msg_handle *msg_handle, int ofd);
 bool tracecmd_msg_done(struct tracecmd_msg_handle *msg_handle);
 void tracecmd_msg_set_done(struct tracecmd_msg_handle *msg_handle);
+int tracecmd_msg_read_options(struct tracecmd_msg_handle *msg_handle,
+			      struct tracecmd_output *handle);
+int tracecmd_msg_send_options(struct tracecmd_msg_handle *msg_handle,
+			      struct tracecmd_output *handle);
 
 int tracecmd_msg_send_trace_req(struct tracecmd_msg_handle *msg_handle,
 				int argc, char **argv, bool use_fifos,
diff --git a/lib/trace-cmd/include/trace-cmd-local.h b/lib/trace-cmd/include/trace-cmd-local.h
index 6ac341374941..cfa3e97ae445 100644
--- a/lib/trace-cmd/include/trace-cmd-local.h
+++ b/lib/trace-cmd/include/trace-cmd-local.h
@@ -94,5 +94,7 @@ int out_write_emty_cpu_data(struct tracecmd_output *handle, int cpus);
 off64_t msg_lseek(struct tracecmd_msg_handle *msg_handle, off64_t offset, int whence);
 unsigned long long get_last_option_offset(struct tracecmd_input *handle);
 unsigned int get_meta_strings_size(struct tracecmd_input *handle);
+int trace_append_options(struct tracecmd_output *handle, void *buf, size_t len);
+void *trace_get_options(struct tracecmd_output *handle, size_t *len);
 
 #endif /* _TRACE_CMD_LOCAL_H */
diff --git a/lib/trace-cmd/trace-msg.c b/lib/trace-cmd/trace-msg.c
index a0c7f1818b0f..9c26401abe66 100644
--- a/lib/trace-cmd/trace-msg.c
+++ b/lib/trace-cmd/trace-msg.c
@@ -108,7 +108,8 @@ struct tracecmd_msg_header {
 	C(TRACE_RESP,	7,	sizeof(struct tracecmd_msg_trace_resp)),\
 	C(CLOSE_RESP,	8,	0),					\
 	C(TIME_SYNC,	9,	sizeof(struct tracecmd_msg_tsync)),	\
-	C(TRACE_PROXY,	10,	sizeof(struct tracecmd_msg_trace_proxy)),
+	C(TRACE_PROXY,	10,	sizeof(struct tracecmd_msg_trace_proxy)), \
+	C(CONT,		11,	0),
 
 #undef C
 #define C(a,b,c)	MSG_##a = b
@@ -783,6 +784,14 @@ int tracecmd_msg_send_close_resp_msg(struct tracecmd_msg_handle *msg_handle)
 	return tracecmd_msg_send(msg_handle, &msg);
 }
 
+int tracecmd_msg_cont(struct tracecmd_msg_handle *msg_handle)
+{
+	struct tracecmd_msg msg;
+
+	tracecmd_msg_init(MSG_CONT, &msg);
+	return tracecmd_msg_send(msg_handle, &msg);
+}
+
 int tracecmd_msg_data_send(struct tracecmd_msg_handle *msg_handle,
 			   const char *buf, int size)
 {
@@ -823,6 +832,42 @@ int tracecmd_msg_data_send(struct tracecmd_msg_handle *msg_handle,
 	return ret;
 }
 
+/**
+ * tracecmd_msg_send_options - Send options over the network
+ * @msg_handle: message handle, holding the communication context
+ * @handle: The output file that has the options to send
+ *
+ * Send options over the network. This is used when the output handle
+ * has more options to send over the network after the trace. Some
+ * options are sent before, and some sent afterward. Since the receiving
+ * side needs to know the location to update the indexes, it will
+ * handle the section header. This just sends out the raw content to
+ * the receiver (requires that both sides have the same endianess, as
+ * no conversion is made of the content of the options).
+ *
+ * Returns 0 on success and -1 on error.
+ */
+int tracecmd_msg_send_options(struct tracecmd_msg_handle *msg_handle,
+			      struct tracecmd_output *handle)
+{
+	struct tracecmd_msg msg;
+	size_t len;
+	void *buf;
+	int ret;
+
+	buf = trace_get_options(handle, &len);
+	if (!buf)
+		return -1;
+
+	ret = tracecmd_msg_data_send(msg_handle, buf, len);
+	free(buf);
+	if (ret < 0)
+		return ret;
+
+	tracecmd_msg_init(MSG_FIN_DATA, &msg);
+	return tracecmd_msg_send(msg_handle, &msg);
+}
+
 /**
  * tracecmd_msg_flush_data - Send the current cache data over the network
  * @msg_handle: message handle, holding the communication context
@@ -856,32 +901,89 @@ int tracecmd_msg_finish_sending_data(struct tracecmd_msg_handle *msg_handle)
 	return 0;
 }
 
+static int read_msg_data(struct tracecmd_msg_handle *msg_handle,
+			 struct tracecmd_msg *msg)
+{
+	int cmd;
+	int ret;
+
+	ret = tracecmd_msg_recv_wait(msg_handle->fd, msg);
+	if (ret < 0) {
+		tracecmd_warning("reading client %d (%s)", ret, strerror(ret));
+		return ret;
+	}
+
+	cmd = ntohl(msg->hdr.cmd);
+	if (cmd == MSG_FIN_DATA) {
+		/* Finish receiving data */
+		return 0;
+	} else if (cmd != MSG_SEND_DATA) {
+		ret = handle_unexpected_msg(msg_handle, msg);
+		if (ret < 0)
+			return -1;
+		return 0;
+	}
+
+	return msg_buf_len(msg);
+}
+
+/**
+ * tracecmd_msg_read_options - Receive options from over the network
+ * @msg_handle: message handle, holding the communication context
+ * @handle: The output file to write the options to.
+ *
+ * Receive the options sent by tracecmd_msg_send_options().
+ * See that function's documentation for mor details.
+ *
+ * Returns 0 on success and -1 on error.
+ */
+int tracecmd_msg_read_options(struct tracecmd_msg_handle *msg_handle,
+			      struct tracecmd_output *handle)
+{
+	struct tracecmd_msg msg;
+	size_t len = 0;
+	void *buf = NULL;
+	void *tmp;
+	int ret;
+	int n;
+
+	memset(&msg, 0, sizeof(msg));
+	while (!tracecmd_msg_done(msg_handle)) {
+		n = read_msg_data(msg_handle, &msg);
+		if (n <= 0)
+			break;
+
+		tmp = realloc(buf, n + len);
+		if (!tmp)
+			goto error;
+		buf = tmp;
+		memcpy(buf + len, msg.buf, n);
+		len += n;
+		msg_free(&msg);
+	}
+	msg_free(&msg);
+
+	ret = trace_append_options(handle, buf, len);
+	free(buf);
+
+	return ret;
+ error:
+	msg_free(&msg);
+	free(buf);
+	return -1;
+}
+
 int tracecmd_msg_read_data(struct tracecmd_msg_handle *msg_handle, int ofd)
 {
 	struct tracecmd_msg msg;
-	int t, n, cmd;
+	int t, n;
 	ssize_t s;
 	int ret;
 
 	while (!tracecmd_msg_done(msg_handle)) {
-		ret = tracecmd_msg_recv_wait(msg_handle->fd, &msg);
-		if (ret < 0) {
-			tracecmd_warning("reading client %d (%s)", ret, strerror(ret));
-			return ret;
-		}
-
-		cmd = ntohl(msg.hdr.cmd);
-		if (cmd == MSG_FIN_DATA) {
-			/* Finish receiving data */
+		n = read_msg_data(msg_handle, &msg);
+		if (n <= 0)
 			break;
-		} else if (cmd != MSG_SEND_DATA) {
-			ret = handle_unexpected_msg(msg_handle, &msg);
-			if (ret < 0)
-				goto error;
-			goto next;
-		}
-
-		n = msg_buf_len(&msg);
 		t = n;
 		s = 0;
 		while (t > 0) {
@@ -896,10 +998,9 @@ int tracecmd_msg_read_data(struct tracecmd_msg_handle *msg_handle, int ofd)
 			t -= s;
 			s = n - t;
 		}
-
-next:
 		msg_free(&msg);
 	}
+	msg_free(&msg);
 
 	return 0;
 
@@ -947,6 +1048,11 @@ error:
 	return ret;
 }
 
+int tracecmd_msg_wait(struct tracecmd_msg_handle *msg_handle)
+{
+	return tracecmd_msg_wait_for_cmd(msg_handle, MSG_CONT);
+}
+
 int tracecmd_msg_wait_close(struct tracecmd_msg_handle *msg_handle)
 {
 	return tracecmd_msg_wait_for_cmd(msg_handle, MSG_CLOSE);
diff --git a/lib/trace-cmd/trace-output.c b/lib/trace-cmd/trace-output.c
index 1fc41424eca7..bdec75d67bc4 100644
--- a/lib/trace-cmd/trace-output.c
+++ b/lib/trace-cmd/trace-output.c
@@ -1902,30 +1902,11 @@ int tracecmd_prepare_options(struct tracecmd_output *handle, off64_t offset, int
 	return curr == -1 ? -1 : 0;
 }
 
-static int write_options(struct tracecmd_output *handle)
+static tsize_t write_options_start(struct tracecmd_output *handle)
 {
-	struct tracecmd_option *options;
-	unsigned long long endian8;
-	unsigned short endian2;
-	unsigned int endian4;
-	bool new = false;
 	tsize_t offset;
 	int ret;
 
-	/* Check if there are unsaved options */
-	list_for_each_entry(options, &handle->options, list) {
-		if (!options->offset) {
-			new = true;
-			break;
-		}
-	}
-	/*
-	 * Even if there are no new options, if options_next is set, it requires
-	 * adding a new empty options section as the previous one already
-	 * points to it.
-	 */
-	if (!new && !handle->options_next)
-		return 0;
 	offset = do_lseek(handle, 0, SEEK_CUR);
 
 	if (handle->options_next) {
@@ -1950,7 +1931,55 @@ static int write_options(struct tracecmd_output *handle)
 			return -1;
 	}
 
-	offset = out_write_section_header(handle, TRACECMD_OPTION_DONE, "options", 0, false);
+	return out_write_section_header(handle, TRACECMD_OPTION_DONE, "options", 0, false);
+}
+
+static tsize_t write_options_end(struct tracecmd_output *handle, tsize_t offset)
+{
+	unsigned long long endian8;
+	unsigned short endian2;
+	unsigned int endian4;
+
+	endian2 = convert_endian_2(handle, TRACECMD_OPTION_DONE);
+	if (do_write_check(handle, &endian2, 2))
+		return -1;
+	endian4 = convert_endian_4(handle, 8);
+	if (do_write_check(handle, &endian4, 4))
+		return -1;
+	endian8 = 0;
+	handle->options_start = do_lseek(handle, 0, SEEK_CUR);
+	if (do_write_check(handle, &endian8, 8))
+		return -1;
+	if (out_update_section_header(handle, offset))
+		return -1;
+
+	return 0;
+}
+
+static int write_options(struct tracecmd_output *handle)
+{
+	struct tracecmd_option *options;
+	unsigned short endian2;
+	unsigned int endian4;
+	bool new = false;
+	tsize_t offset;
+
+	/* Check if there are unsaved options */
+	list_for_each_entry(options, &handle->options, list) {
+		if (!options->offset) {
+			new = true;
+			break;
+		}
+	}
+	/*
+	 * Even if there are no new options, if options_next is set, it requires
+	 * adding a new empty options section as the previous one already
+	 * points to it.
+	 */
+	if (!new && !handle->options_next)
+		return 0;
+
+	offset = write_options_start(handle);
 	if (offset == (off_t)-1)
 		return -1;
 
@@ -1970,20 +1999,104 @@ static int write_options(struct tracecmd_output *handle)
 			return -1;
 	}
 
-	endian2 = convert_endian_2(handle, TRACECMD_OPTION_DONE);
-	if (do_write_check(handle, &endian2, 2))
-		return -1;
-	endian4 = convert_endian_4(handle, 8);
-	if (do_write_check(handle, &endian4, 4))
-		return -1;
-	endian8 = 0;
-	handle->options_start = do_lseek(handle, 0, SEEK_CUR);
-	if (do_write_check(handle, &endian8, 8))
+	return write_options_end(handle, offset);
+}
+
+/**
+ * trace_get_options - Get the current options from the output file handle
+ * @handle: The output file descriptor that has options.
+ * @len: Returns the length of the buffer allocated and returned.
+ *
+ * Reads the options that have not been written to the file yet,
+ * puts them into an allocated buffer and sets @len to the size
+ * added. Used by trace-msg.c to send options over the network.
+ *
+ * Note, the options cannot be referenced again once this is called.
+ *  New options can be added and referenced.
+ *
+ * Returns an allocated buffer (must be freed with free()) that contains
+ *   the options to send, with @len set to the size of the content.
+ *   NULL on error (and @len is undefined).
+ */
+__hidden void *trace_get_options(struct tracecmd_output *handle, size_t *len)
+{
+	struct tracecmd_msg_handle msg_handle;
+	struct tracecmd_output out_handle;
+	struct tracecmd_option *options;
+	unsigned short endian2;
+	unsigned int endian4;
+	tsize_t offset;
+	void *buf = NULL;
+
+	/* Use the msg_cache as our output */
+	memset(&msg_handle, 0, sizeof(msg_handle));
+	msg_handle.cfd = -1;
+	if (tracecmd_msg_handle_cache(&msg_handle) < 0)
+		return NULL;
+
+	out_handle = *handle;
+	out_handle.fd = msg_handle.cfd;
+	out_handle.msg_handle = &msg_handle;
+
+	list_for_each_entry(options, &handle->options, list) {
+		/* Option is already saved, skip it */
+		if (options->offset)
+			continue;
+		endian2 = convert_endian_2(handle, options->id);
+		if (do_write_check(&out_handle, &endian2, 2))
+			goto out;
+		endian4 = convert_endian_4(handle, options->size);
+		if (do_write_check(&out_handle, &endian4, 4))
+			goto out;
+		/* The option can not be referenced again */
+		options->offset = -1;
+		if (do_write_check(&out_handle, options->data, options->size))
+			goto out;
+	}
+
+	offset = do_lseek(&out_handle, 0, SEEK_CUR);
+	buf = malloc(offset);
+	if (!buf)
+		goto out;
+
+	if (do_lseek(&out_handle, 0, SEEK_SET) == (off64_t)-1)
+		goto out;
+	*len = read(msg_handle.cfd, buf, offset);
+	if (*len != offset) {
+		free(buf);
+		buf = NULL;
+	}
+
+ out:
+	close(msg_handle.cfd);
+	return buf;
+}
+
+/**
+ * trace_append_options - Append options to the file
+ * @handle: The output file descriptor that has options.
+ * @buf: The options to append.
+ * @len: The length of @buf.
+ *
+ * Will add an options section header for the content of @buf to
+ * be written as options into the @handle.
+ * Used by trace-msg.c to retrieve options over the network.
+ *
+ * Returns 0 on success and -1 on error.
+ */
+__hidden int trace_append_options(struct tracecmd_output *handle, void *buf,
+				  size_t len)
+{
+	tsize_t offset;
+
+	offset = write_options_start(handle);
+	if (offset == (off_t)-1)
 		return -1;
-	if (out_update_section_header(handle, offset))
+
+	if (do_write_check(handle, buf, len))
 		return -1;
 
-	return 0;
+	return write_options_end(handle, offset);
 }
 
 int tracecmd_write_meta_strings(struct tracecmd_output *handle)
diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index a6829565a679..e0d2f2b0f554 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -739,8 +739,10 @@ static void tell_guests_to_stop(struct common_record_context *ctx)
 	/* Wait for guests to acknowledge */
 	for_all_instances(instance) {
 		if (is_guest(instance)) {
-			tracecmd_msg_wait_close_resp(instance->msg_handle);
-			tracecmd_msg_handle_close(instance->msg_handle);
+			if (!is_proxy(instance)) {
+				tracecmd_msg_wait_close_resp(instance->msg_handle);
+				tracecmd_msg_handle_close(instance->msg_handle);
+			}
 		}
 	}
 }
@@ -4366,15 +4368,18 @@ static void record_data(struct common_record_context *ctx)
 {
 	struct tracecmd_output *handle;
 	struct buffer_instance *instance;
+	bool have_proxy = false;
 	bool local = false;
 	int max_cpu_count = local_cpu_count;
 	char **temp_files;
 	int i;
 
 	for_all_instances(instance) {
-		if (is_guest(instance))
+		if (is_guest(instance)) {
 			write_guest_file(instance);
-		else if (host && instance->msg_handle)
+			if (is_proxy(instance))
+				have_proxy = true;
+		} else if (host && instance->msg_handle)
 			finish_network(instance->msg_handle);
 		else
 			local = true;
@@ -4423,6 +4428,22 @@ static void record_data(struct common_record_context *ctx)
 
 		add_options(handle, ctx);
 
+		/*
+		 * If we connected to a proxy, then it will now send us
+		 * the tsync data for our file.
+		 */
+		if (have_proxy) {
+			for_all_instances(instance) {
+				if (!is_proxy(instance))
+					continue;
+				/* Tell proxy we are ready for the rest */
+				tracecmd_msg_cont(instance->msg_handle);
+				tracecmd_msg_read_options(instance->msg_handle, handle);
+				tracecmd_msg_wait_close_resp(instance->msg_handle);
+				tracecmd_msg_handle_close(instance->msg_handle);
+			}
+		}
+
 		/* Only record the top instance under TRACECMD_OPTION_CPUSTAT*/
 		if (!no_top_instance() && !top_instance.msg_handle) {
 			struct trace_seq *s = top_instance.s_save;
@@ -6680,6 +6701,14 @@ static void finalize_record_trace(struct common_record_context *ctx)
 		if (instance->flags & BUFFER_FL_KEEP)
 			write_tracing_on(instance,
 					 instance->tracing_on_init_val);
+		if (is_proxy_server(instance) && instance->network_handle) {
+			/* Now wait for the recorder to be ready for us to send more */
+			tracecmd_msg_wait(ctx->instance->msg_handle);
+			if (ctx->tsc2nsec.mult)
+				add_tsc2nsec(ctx->instance->network_handle, &ctx->tsc2nsec);
+			tracecmd_msg_send_options(ctx->instance->msg_handle,
+						  ctx->instance->network_handle);
+		}
 		if (is_agent(instance)) {
 			tracecmd_msg_send_close_resp_msg(instance->msg_handle);
 			tracecmd_output_close(instance->network_handle);
@@ -6916,6 +6945,10 @@ static void record_trace(int argc, char **argv,
 		wait_threads();
 
 	if (is_proxy_server(ctx->instance) && ctx->instance->network_handle) {
+		tracecmd_tsync_with_guest_stop(ctx->instance->tsync);
+		trace_add_guest_info(ctx->instance->network_handle, ctx->instance);
+		if (ctx->tsc2nsec.mult)
+			add_tsc2nsec(ctx->instance->network_handle, &ctx->tsc2nsec);
 		tracecmd_write_options(ctx->instance->network_handle);
 		tracecmd_write_meta_strings(ctx->instance->network_handle);
 		tracecmd_msg_finish_sending_data(ctx->instance->msg_handle);
-- 
2.35.1

[PATCH 25/26] trace-cmd: Have the guest structure hold guest trace_id

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

Have the guest structure hold the guest's trace_id and also use the guest
structure for the CPU counts in mapping the guest CPUs to pids. For the
agent proxy, the instance will be of the host and not the guest, so it
makes more sense to use the guest structure instead.

Also clean up the trace_add_guest_info() call where the network uses the
instance, and the cpus use the max cpus.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/include/trace-local.h |  1 +
 tracecmd/trace-agent.c         | 10 ++++++++--
 tracecmd/trace-vm.c            | 30 +++++++++++++++++++++---------
 3 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/tracecmd/include/trace-local.h b/tracecmd/include/trace-local.h
index 295be5dda7b5..a86d22ca9ad8 100644
--- a/tracecmd/include/trace-local.h
+++ b/tracecmd/include/trace-local.h
@@ -335,6 +335,7 @@ void show_options(const char *prefix, struct buffer_instance *buffer);
 struct trace_guest {
 	struct tracefs_instance *instance;
 	char *name;
+	unsigned long long trace_id;
 	int cid;
 	int pid;
 	int cpu_max;
diff --git a/tracecmd/trace-agent.c b/tracecmd/trace-agent.c
index 4430cb5506ff..3098a1ea2b8b 100644
--- a/tracecmd/trace-agent.c
+++ b/tracecmd/trace-agent.c
@@ -153,6 +153,7 @@ static void agent_handle(int sd, int nr_cpus, int page_size,
 	struct tracecmd_time_sync *tsync = NULL;
 	struct tracecmd_msg_handle *msg_handle;
 	const char *tsync_proto = NULL;
+	struct trace_guest *guest;
 	unsigned long long peer_trace_id;
 	unsigned long long trace_id;
 	unsigned long flags = rcid >= 0 ? TRACECMD_MSG_FL_PROXY : 0;
@@ -178,16 +179,21 @@ static void agent_handle(int sd, int nr_cpus, int page_size,
 	if (!msg_handle)
 		die("Failed to allocate message handle");
 
-	if (rcid >= 0)
+	if (rcid >= 0) {
 		ret = tracecmd_msg_recv_trace_proxy(msg_handle, &argc, &argv,
 						    &use_fifos, &peer_trace_id,
 						    &tsync_protos,
 						    &client_cpus,
 						    &guests);
-	else
+		/* Update the guests peer_trace_id */
+		guest = trace_get_guest(rcid, NULL);
+		if (guest)
+			guest->trace_id = peer_trace_id;
+	} else {
 		ret = tracecmd_msg_recv_trace_req(msg_handle, &argc, &argv,
 						  &use_fifos, &peer_trace_id,
 						  &tsync_protos);
+	}
 	if (ret < 0)
 		die("Failed to receive trace request");
 
diff --git a/tracecmd/trace-vm.c b/tracecmd/trace-vm.c
index 84f28824e3c5..09bd60259258 100644
--- a/tracecmd/trace-vm.c
+++ b/tracecmd/trace-vm.c
@@ -434,26 +434,39 @@ int get_guest_vcpu_pid(unsigned int guest_cid, unsigned int guest_vcpu)
 void
 trace_add_guest_info(struct tracecmd_output *handle, struct buffer_instance *instance)
 {
+	unsigned long long trace_id;
 	struct trace_guest *guest;
 	const char *name;
 	char *buf, *p;
+	int cpus;
 	int size;
 	int pid;
 	int i;
 
 	if (is_network(instance)) {
 		name = instance->name;
+		cpus = instance->cpu_count;
+		trace_id = instance->trace_id;
 	} else {
 		guest = trace_get_guest(instance->cid, NULL);
 		if (!guest)
 			return;
+		cpus = guest->cpu_max;
 		name = guest->name;
+		/*
+		 * If this is a proxy, the trace_id of the guest is
+		 * in the guest descriptor (added in trace_tsync_as_host().
+		 */
+		if (guest->trace_id)
+			trace_id = guest->trace_id;
+		else
+			trace_id = instance->trace_id;
 	}
 
 	size = strlen(name) + 1;
 	size += sizeof(long long);	/* trace_id */
 	size += sizeof(int);		/* cpu count */
-	size += instance->cpu_count * 2 * sizeof(int);	/* cpu,pid pair */
+	size += cpus * 2 * sizeof(int);	/* cpu,pid pair */
 
 	buf = calloc(1, size);
 	if (!buf)
@@ -462,17 +475,16 @@ trace_add_guest_info(struct tracecmd_output *handle, struct buffer_instance *ins
 	strcpy(p, name);
 	p += strlen(name) + 1;
 
-	memcpy(p, &instance->trace_id, sizeof(long long));
+	memcpy(p, &trace_id, sizeof(long long));
 	p += sizeof(long long);
 
-	memcpy(p, &instance->cpu_count, sizeof(int));
+	memcpy(p, &cpus, sizeof(int));
 	p += sizeof(int);
-	for (i = 0; i < instance->cpu_count; i++) {
-		pid = -1;
-		if (!is_network(instance)) {
-			if (i < guest->cpu_max)
-				pid = guest->cpu_pid[i];
-		}
+	for (i = 0; i < cpus; i++) {
+		if (is_network(instance))
+			pid = -1;
+		else
+			pid = guest->cpu_pid[i];
 		memcpy(p, &i, sizeof(int));
 		p += sizeof(int);
 		memcpy(p, &pid, sizeof(int));
-- 
2.35.1

[PATCH 26/26] trace-cmd: Have the host agent proxy control the time synchronization

[Steven Rostedt]

From: "Steven Rostedt (Google)" <rostedt@goodmis.org>

The time synchronization logic between agent and recorder made the
assumption that the host is running the recorder and the guest is
running the agent. This is very limiting, especially for environments
where you have a streamlined host that has no disk space, and a guest
that is "privileged".

Have the agent proxy handle the time synchronization for the host, and
the trace-cmd record running on the guest retrieve this information from
the host.

Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
---
 tracecmd/include/trace-local.h |  5 ++--
 tracecmd/trace-agent.c         | 17 +++++++++---
 tracecmd/trace-record.c        | 47 ++++++++++++++++++++++++++--------
 tracecmd/trace-tsync.c         | 12 +++++----
 4 files changed, 60 insertions(+), 21 deletions(-)

diff --git a/tracecmd/include/trace-local.h b/tracecmd/include/trace-local.h
index a86d22ca9ad8..3d025f86fb19 100644
--- a/tracecmd/include/trace-local.h
+++ b/tracecmd/include/trace-local.h
@@ -121,8 +121,9 @@ void trace_convert(int argc, char **argv);
 
 int trace_record_agent(struct tracecmd_msg_handle *msg_handle,
 		       int cpus, int *fds,
-		       int argc, char **argv, bool use_fifos,
-		       unsigned long long trace_id, const char *host);
+		       int argc, char **argv,
+		       bool use_fifos, struct tracecmd_time_sync *tsync,
+		       unsigned long long trace_id, int rcid, const char *host);
 
 struct hook_list;
 
diff --git a/tracecmd/trace-agent.c b/tracecmd/trace-agent.c
index 3098a1ea2b8b..ea41ad540701 100644
--- a/tracecmd/trace-agent.c
+++ b/tracecmd/trace-agent.c
@@ -238,17 +238,26 @@ static void agent_handle(int sd, int nr_cpus, int page_size,
 
 	if (tsync_proto) {
 		fd = wait_for_connection(fd);
-		tsync = trace_tsync_as_guest(fd, tsync_proto, get_clock(argc, argv),
-					     remote_id, local_id);
+
+		if (rcid >= 0) {
+			tsync = trace_tsync_as_host(fd, trace_id, 0, rcid,
+						    client_cpus, tsync_proto,
+						    get_clock(argc, argv));
+		} else {
+			tsync = trace_tsync_as_guest(fd, tsync_proto,
+						     get_clock(argc, argv),
+						     remote_id, local_id);
+		}
 		if (!tsync)
 			close(fd);
 	}
 
 	trace_record_agent(msg_handle, nr_cpus, fds, argc, argv,
-			   use_fifos, trace_id, network);
+			   use_fifos, tsync, trace_id, rcid, network);
 
 	if (tsync) {
-		tracecmd_tsync_with_host_stop(tsync);
+		if (rcid < 0)
+			tracecmd_tsync_with_host_stop(tsync);
 		tracecmd_tsync_free(tsync);
 	}
 
diff --git a/tracecmd/trace-record.c b/tracecmd/trace-record.c
index e0d2f2b0f554..541bce7298ce 100644
--- a/tracecmd/trace-record.c
+++ b/tracecmd/trace-record.c
@@ -686,6 +686,12 @@ add_tsc2nsec(struct tracecmd_output *handle, struct tsc_nsec *tsc2nsec)
 	tracecmd_add_option_v(handle, TRACECMD_OPTION_TSC2NSEC, vector, 3);
 }
 
+static void guest_tsync_complete(struct buffer_instance *instance)
+{
+	tracecmd_tsync_with_host_stop(instance->tsync);
+	tracecmd_tsync_free(instance->tsync);
+}
+
 static void host_tsync_complete(struct common_record_context *ctx,
 				struct buffer_instance *instance)
 {
@@ -732,8 +738,12 @@ static void tell_guests_to_stop(struct common_record_context *ctx)
 	}
 
 	for_all_instances(instance) {
-		if (is_guest(instance))
-			host_tsync_complete(ctx, instance);
+		if (is_guest(instance)) {
+			if (is_proxy(instance))
+				guest_tsync_complete(instance);
+			else
+				host_tsync_complete(ctx, instance);
+		}
 	}
 
 	/* Wait for guests to acknowledge */
@@ -3851,11 +3861,15 @@ static int host_tsync(struct common_record_context *ctx,
 		fd = trace_vsock_open(instance->cid, tsync_port);
 	}
 
-	instance->tsync = trace_tsync_as_host(fd, top_instance.trace_id,
-					      instance->tsync_loop_interval,
-					      guest_id, instance->cpu_count,
-					      proto, ctx->clock);
-
+	if (is_proxy(instance)) {
+		instance->tsync = trace_tsync_as_guest(fd, proto, ctx->clock,
+						       guest_id, -1);
+	} else {
+		instance->tsync = trace_tsync_as_host(fd, top_instance.trace_id,
+						      instance->tsync_loop_interval,
+						      guest_id, instance->cpu_count,
+						      proto, ctx->clock);
+	}
 	return instance->tsync ? 0 : -1;
 }
 
@@ -4267,6 +4281,7 @@ enum {
 	DATA_FL_DATE		= 1,
 	DATA_FL_OFFSET		= 2,
 	DATA_FL_GUEST		= 4,
+	DATA_FL_PROXY		= 8,
 };
 
 static void add_options(struct tracecmd_output *handle, struct common_record_context *ctx)
@@ -6235,6 +6250,11 @@ static void parse_record_options(int argc,
 			ctx->instance->name = name;
 			add_instance(ctx->instance, 0);
 			ctx->data_flags |= DATA_FL_GUEST;
+
+			/* Do not send a clock to a proxy */
+			if (is_proxy)
+				ctx->instance->flags |= BUFFER_FL_HAS_CLOCK;
+
 			break;
 		}
 		case 'F':
@@ -6706,6 +6726,8 @@ static void finalize_record_trace(struct common_record_context *ctx)
 			tracecmd_msg_wait(ctx->instance->msg_handle);
 			if (ctx->tsc2nsec.mult)
 				add_tsc2nsec(ctx->instance->network_handle, &ctx->tsc2nsec);
+			tracecmd_write_guest_time_shift(ctx->instance->network_handle,
+							ctx->instance->tsync);
 			tracecmd_msg_send_options(ctx->instance->msg_handle,
 						  ctx->instance->network_handle);
 		}
@@ -6821,7 +6843,7 @@ static void record_trace(int argc, char **argv,
 	if (!ctx->output)
 		ctx->output = DEFAULT_INPUT_FILE;
 
-	if (ctx->data_flags & DATA_FL_GUEST)
+	if (ctx->data_flags & (DATA_FL_GUEST | DATA_FL_PROXY))
 		set_tsync_params(ctx);
 
 	make_instances();
@@ -7129,8 +7151,9 @@ void trace_record(int argc, char **argv)
  */
 int trace_record_agent(struct tracecmd_msg_handle *msg_handle,
 		       int cpus, int *fds,
-		       int argc, char **argv, bool use_fifos,
-		       unsigned long long trace_id, const char *host)
+		       int argc, char **argv,
+		       bool use_fifos, struct tracecmd_time_sync *tsync,
+		       unsigned long long trace_id, int rcid, const char *host)
 {
 	struct common_record_context ctx;
 	char **argv_plus;
@@ -7158,8 +7181,12 @@ int trace_record_agent(struct tracecmd_msg_handle *msg_handle,
 	ctx.instance->fds = fds;
 	ctx.instance->use_fifos = use_fifos;
 	ctx.instance->flags |= BUFFER_FL_AGENT;
+	if (rcid >= 0)
+		ctx.data_flags |= DATA_FL_PROXY;
 	ctx.instance->msg_handle = msg_handle;
 	ctx.instance->host = host;
+	ctx.instance->tsync = tsync;
+	ctx.instance->cid = rcid;
 	msg_handle->version = V3_PROTOCOL;
 	top_instance.trace_id = trace_id;
 	record_trace(argc, argv, &ctx);
diff --git a/tracecmd/trace-tsync.c b/tracecmd/trace-tsync.c
index 45ebb9c66a19..a96b4e43e0d9 100644
--- a/tracecmd/trace-tsync.c
+++ b/tracecmd/trace-tsync.c
@@ -228,15 +228,17 @@ struct tracecmd_time_sync *
 trace_tsync_as_guest(int fd, const char *tsync_proto, const char *clock,
 	       unsigned int remote_id, unsigned int local_id)
 {
-	struct tracecmd_time_sync *tsync;
+	struct tracecmd_time_sync *tsync = NULL;
 
 	if (fd < 0)
-		return NULL;
+		 return NULL;
 
-	tsync = tracecmd_tsync_with_host(fd, tsync_proto, clock,
-					 remote_id, local_id);
-	if (!tsync)
+	tsync = tracecmd_tsync_with_host(fd, tsync_proto,
+					 clock, remote_id, local_id);
+	if (!tsync) {
 		warning("Failed to negotiate timestamps synchronization with the host");
+		return NULL;
+	}
 
 	return tsync;
 }
-- 
2.35.1