[RFC PATCH v2 0/4] trace-cruncher: ftrace uprobes support

[RFC PATCH v2 0/4] trace-cruncher: ftrace uprobes support

[Tzvetomir Stoyanov (VMware)]

Ftrace uprobes can be used to create dynamic events for user functions.
Tracing user function can be very useful functionality for trace-cruncher.

v2 changes:
 - Moved the uprobe C code into the ftrace-utils.c file.
 - Reused internal functions from ftrace-util code.
 - Renamed utrace start and stop to enable and disable, to be consistent
   with the other APIs.
 - Do not call utrace disable in the urtace example, as this logic is
   triggered by the destructor.
 - Changed the logic for allocation of internal dynamic array - double the
   array size on each reallocation, instead of increasing the size with
   hard-coded constant.
 - Added support for running a program with arguments for tracing.
 - Do not run the program for tracing in a separate shell.
 - Coding style fixes and optimizations.

Tzvetomir Stoyanov (VMware) (4):
  trace-cruncher: Logic for resolving address to function name
  trace-cruncher: ftrace uprobe raw API
  trace-cruncher: High level wrappers for ftrace uprobes
  trace-cruncher: Example script for uprobes high level API

 examples/user_trace.py |  43 ++
 setup.py               |   4 +-
 src/ftracepy-utils.c   | 676 ++++++++++++++++++++++++++++
 src/ftracepy-utils.h   |  21 +
 src/ftracepy.c         |  45 ++
 src/trace-obj-debug.c  | 982 +++++++++++++++++++++++++++++++++++++++++
 src/trace-obj-debug.h  |  54 +++
 7 files changed, 1823 insertions(+), 2 deletions(-)
 create mode 100755 examples/user_trace.py
 create mode 100644 src/trace-obj-debug.c
 create mode 100644 src/trace-obj-debug.h

-- 
2.35.1

[RFC PATCH v2 1/4] trace-cruncher: Logic for resolving address to function name

[Tzvetomir Stoyanov (VMware)]

Resolving virtual address to function name and vise versa is useful
functionality for a trace application. Trace-cruncher can use it in two
use cases:
 - Resolving VMA to function name, when collecting user application
   performance traces with perf.
 - Resolving function name to VMA, when using ftarce uprobe dynamic
   events.

Proposed implementation uses the bfd library to parse the binary files
and read the symbol table. This information is available only if the
files are not stripped.

Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
 src/trace-obj-debug.c | 982 ++++++++++++++++++++++++++++++++++++++++++
 src/trace-obj-debug.h |  54 +++
 2 files changed, 1036 insertions(+)
 create mode 100644 src/trace-obj-debug.c
 create mode 100644 src/trace-obj-debug.h

diff --git a/src/trace-obj-debug.c b/src/trace-obj-debug.c
new file mode 100644
index 0000000..a6da22f
--- /dev/null
+++ b/src/trace-obj-debug.c
@@ -0,0 +1,982 @@
+// SPDX-License-Identifier: LGPL-2.1
+/*
+ * Copyright (C) 2020, VMware, Tzvetomir Stoyanov <tz.stoyanov@gmail.com>
+ *
+ */
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <limits.h>
+#include <errno.h>
+#include <bfd.h>
+#include <unistd.h>
+#include <dlfcn.h>
+#include <fnmatch.h>
+#include <ctype.h>
+
+#include "trace-obj-debug.h"
+
+//#define DEBUG_INTERNALS
+
+/* Got from demangle.h */
+#define DMGL_AUTO (1 << 8)
+
+struct debug_bfd_handle {
+	bfd *bfd;
+	unsigned long long addr_offset;
+};
+
+enum match_type {
+	MATCH_EXACT	= 0,
+	MATH_WILDCARD	= 1,
+};
+
+struct debug_symbols {
+	struct debug_symbols		*next;
+	struct dbg_trace_symbols	symbol;
+	enum match_type			match;
+};
+
+struct debug_file {
+	struct debug_file		*next;
+	char				*file_name;
+
+	/* Start and end address, where this file is mapped into the memory of the process. */
+	unsigned long long		vmem_start;
+	unsigned long long		vmem_end;
+
+	/* bfd library context for this file. */
+	struct debug_bfd_handle	*dbg;
+
+	/* Symbols to resolve, search in this file only. */
+	int				sym_count;
+	struct debug_symbols		*sym;
+};
+
+struct dbg_trace_context {
+	/* PID or name of the process. */
+	int				pid;
+	char				*fname;
+
+	/* List of all libraries and object files, mapped in the memory of the process. */
+	struct dbg_trace_pid_maps	*fmaps;
+
+	/* Symbols to resolve, search in all files. */
+	int				sym_count;
+	struct debug_symbols		*sym;
+
+	/* List of all libraries and object files, opened from bfd library for processing. */
+	struct debug_file		*files;
+};
+
+#define RESOLVE_NAME		(1 << 0)
+#define RESOLVE_VMA		(1 << 1)
+#define RESOLVE_FOFFSET		(1 << 2)
+
+struct debug_obj_job {
+	unsigned int flags;
+	unsigned long long addr_offset;
+	struct debug_symbols *symbols;
+};
+
+struct debug_dwarf_bfd_context {
+	asymbol **table;
+	struct debug_obj_job *job;
+};
+
+/*
+ * A hook, called from bfd library for each section of the file.
+ * The logic is used for symbol name and file offset resolving from given symbol VMA.
+ */
+static void process_bfd_section(bfd *abfd, asection *section, void *param)
+{
+	struct debug_dwarf_bfd_context *context = (struct debug_dwarf_bfd_context *)param;
+	unsigned int discriminator;
+	const char *functionname;
+	struct debug_symbols *s;
+	unsigned long long vma;
+	const char *filename;
+	unsigned int line;
+	bfd_boolean found;
+
+	/* Skip sections that have no code. */
+	if (!(section->flags & SEC_CODE))
+		return;
+
+	/* Loop through all symbols, that have to be resolved. */
+	for (s = context->job->symbols; s; s = s->next) {
+		if (s->symbol.vma_near)
+			vma = s->symbol.vma_near;
+		else if (s->symbol.vma_start)
+			vma = s->symbol.vma_start;
+		else
+			continue;
+
+		/* Adjust the symbol's VMA, if this section is dynamically loaded. */
+		if (abfd->flags & DYNAMIC)
+			vma -=  context->job->addr_offset;
+
+		/* Check if requested symbol's vma is within the current section. */
+		if (vma && section->vma <= vma &&
+		    (section->vma + section->size) > vma) {
+
+			/* Found the file, where the symbol is defined. */
+			if (!s->symbol.fname)
+				s->symbol.fname = strdup(abfd->filename);
+
+			/* Found the offset of the symbol within the file. */
+			if (context->job->flags & RESOLVE_FOFFSET)
+				s->symbol.foffset = section->filepos + (vma - section->vma);
+
+			/* Look for the nearest function. */
+			if (!s->symbol.name && (context->job->flags & RESOLVE_NAME)) {
+				found = bfd_find_nearest_line_discriminator(abfd, section, context->table,
+									    vma - section->vma, &filename,
+									    &functionname, &line, &discriminator);
+#ifdef DEBUG_INTERNALS
+				printf("Find addr near 0x%X, offset 0x%X - > vma - 0x%X in %s, found %s\n\r",
+					s->symbol.vma_near, context->job->addr_offset, vma, abfd->filename,
+					found ? functionname : "NA");
+#endif
+				if (found) {
+					/* Demangle the name of the function. */
+					s->symbol.name = bfd_demangle(abfd, functionname, DMGL_AUTO);
+					/* Found the name of the symbol. */
+					if (!s->symbol.name)
+						s->symbol.name = strdup(functionname);
+				}
+			}
+		}
+	}
+}
+
+/* Load the symbol table from the file. */
+static asymbol **get_sym_table(bfd *handle)
+{
+	long size, ssize, dsize;
+	asymbol **symtable;
+	long count;
+
+	if ((bfd_get_file_flags(handle) & HAS_SYMS) == 0)
+		return NULL;
+
+	dsize = bfd_get_dynamic_symtab_upper_bound(handle);
+	size = dsize > 0 ? dsize : 0;
+
+	ssize = bfd_get_symtab_upper_bound(handle);
+	size += ssize > 0 ? ssize : 0;
+
+	if (size <= 0)
+		return NULL;
+
+	symtable = (asymbol **) calloc(1, size);
+	if (!symtable)
+		return NULL;
+
+	count = bfd_canonicalize_symtab(handle, symtable);
+	count += bfd_canonicalize_dynamic_symtab(handle, symtable + count);
+	if (count <= 0) {
+		free(symtable);
+		return NULL;
+	}
+
+	return symtable;
+}
+
+/* Match the requested name to the name of the symbol. Handle a wildcard match. */
+static bool symbol_match(char *pattern, enum match_type match, const char *symbol)
+{
+	bool ret = false;
+
+	switch (match) {
+	case MATCH_EXACT:
+		if (strlen(pattern) == strlen(symbol) &&
+		    !strcmp(pattern, symbol))
+			ret = true;
+		break;
+	case MATH_WILDCARD:
+		if (!fnmatch(pattern, symbol, 0))
+			ret = true;
+		break;
+	}
+
+	return ret;
+}
+
+/* Lookup in the file's symbol table.
+ * The logic is used for symbol VMA resolving from given symbol name.
+ */
+static int lookup_bfd_sym(struct debug_dwarf_bfd_context *context)
+{
+	struct debug_symbols *s, *last = NULL;
+	struct debug_symbols *new, *new_list = NULL;
+	unsigned long long vma;
+	asymbol **sp;
+	int res = 0;
+
+	for (sp = context->table; *sp != NULL; sp++) {
+		/* Skip the symbol, if it is not a function. */
+		if (!((*sp)->flags & BSF_FUNCTION))
+			continue;
+		/* Loop through all symbols that should be resolved. */
+		for (s = context->job->symbols; s; s = s->next) {
+			if (!s->symbol.name)
+				continue;
+			last = s;
+			if (!symbol_match(s->symbol.name, s->match, (*sp)->name))
+				continue;
+#ifdef DEBUG_INTERNALS
+			printf("Matched %s, pattern %s\n\r", (*sp)->name, s->symbol.name);
+#endif
+			vma = (*sp)->value + (*sp)->section->vma;
+			/* Adjust the VMA, if the section is dynamically loaded. */
+			if ((*sp)->the_bfd->flags & DYNAMIC)
+				vma += context->job->addr_offset;
+			if (s->match == MATCH_EXACT) {
+				/* Exact match, update the VMA. */
+				s->symbol.vma_start = vma;
+			} else if (s->match == MATH_WILDCARD) {
+				/* Wildcard pattern match, create a new symbol. */
+				new = calloc(1, sizeof(struct debug_symbols));
+				if (!new)
+					break;
+				new->symbol.name = strdup((*sp)->name);
+				new->symbol.vma_start = vma;
+				new->symbol.vma_near = s->symbol.vma_near;
+				new->symbol.foffset = s->symbol.foffset;
+				new->symbol.cookie = s->symbol.cookie;
+				if (s->symbol.fname)
+					new->symbol.fname = strdup(s->symbol.fname);
+				new->next = new_list;
+				new_list = new;
+			}
+			res++;
+		}
+	}
+	if (last && !last->next)
+		last->next = new_list;
+
+	return res;
+}
+
+/* Process a bfd object from the file. */
+static int process_bfd_object(bfd *abfd, struct debug_obj_job *job)
+{
+	struct debug_dwarf_bfd_context context;
+	int ret = 0;
+
+	memset(&context, 0, sizeof(context));
+	context.job = job;
+	if (bfd_check_format_matches(abfd, bfd_object, NULL) ||
+	    bfd_check_format_matches(abfd, bfd_core, NULL)) {
+		context.table = get_sym_table(abfd);
+		if (!context.table)
+			return -1;
+
+		/* Resolve VMA from the symbol table. */
+		if (job->flags & RESOLVE_VMA)
+			lookup_bfd_sym(&context);
+
+		/* Resolve symbol name and file offset from file's sections. */
+		if ((job->flags & RESOLVE_NAME) || (job->flags & RESOLVE_FOFFSET))
+			bfd_map_over_sections(abfd, process_bfd_section, &context);
+
+		free(context.table);
+	} else {
+		ret = -1;
+	}
+
+	return ret;
+}
+
+/* Open a bfd archive file and read all objects. */
+static int read_all_bfd(bfd *abfd, struct debug_obj_job *job)
+{
+	bfd *last_arfile = NULL;
+	bfd *arfile = NULL;
+	int ret = 0;
+
+	if (bfd_check_format(abfd, bfd_archive)) {
+		for (;;) {
+			bfd_set_error(bfd_error_no_error);
+			arfile = bfd_openr_next_archived_file(abfd, arfile);
+			if (!arfile) {
+				if (bfd_get_error() != bfd_error_no_more_archived_files)
+					break;
+			}
+			ret = read_all_bfd(arfile, job);
+			if (last_arfile)
+				bfd_close(last_arfile);
+			last_arfile = arfile;
+		}
+		if (last_arfile)
+			bfd_close(last_arfile);
+	} else
+		ret = process_bfd_object(abfd, job);
+
+	return ret;
+}
+
+/**
+ * resolve_symbol_vma - name -> (vma, file offset) resolving
+ * @obj - pointer to object, returned by trace_obj_debug_create()
+ * @symbols - link list with desired symbols, with given name
+ *
+ * Get VMA and file offset of the symbols with given name.
+ * Return 0 on success, -1 on error.
+ */
+static int resolve_symbol_vma(struct debug_bfd_handle *obj,
+			      struct debug_symbols *symbols)
+{
+	struct debug_obj_job job;
+	int ret;
+
+	memset(&job, 0, sizeof(job));
+	job.flags |= RESOLVE_VMA;
+	job.flags |= RESOLVE_FOFFSET;
+	job.symbols = symbols;
+	job.addr_offset = obj->addr_offset;
+	ret = read_all_bfd(obj->bfd, &job);
+
+	return ret;
+}
+
+/**
+ * resolve_symbol_name - vma -> name resolving
+ * @obj - pointer to object, returned by trace_obj_debug_create()
+ * @symbols - link list with desired symbols, with given VMA
+ *
+ * Get names of the symbols with given VMA, look for nearest symbol to that VMA.
+ * Return 0 on success, -1 on error.
+ */
+static int resolve_symbol_name(struct debug_bfd_handle *obj,
+			       struct debug_symbols *symbols)
+{
+	struct debug_obj_job job;
+
+	if (!obj || !obj->bfd)
+		return -1;
+	memset(&job, 0, sizeof(job));
+	job.flags |= RESOLVE_NAME;
+	job.addr_offset = obj->addr_offset;
+	job.symbols = symbols;
+	return read_all_bfd(obj->bfd, &job);
+}
+
+/**
+ * debug_handle_destroy - Close file opened with trace_obj_debug_create()
+ * @obj - pointer to object, returned by trace_obj_debug_create()
+ *
+ * Close the file and free any allocated resources, related to file's debug
+ * information.
+ */
+static void debug_handle_destroy(struct debug_bfd_handle *obj)
+{
+	if (obj && obj->bfd)
+		bfd_close(obj->bfd);
+	free(obj);
+}
+
+/**
+ * debug_handle_create - Open binary file for parsing ELF and DWARF information
+ * @name: Name of the binary ELF file.
+ *
+ * Return pointer to trace_obj_debug structure, that can be passed to other APIs
+ * for extracting debug information from the file. NULL in case of an error.
+ */
+static struct debug_bfd_handle *debug_handle_create(char *file)
+{
+	struct debug_bfd_handle *obj = NULL;
+
+	obj = calloc(1, sizeof(*obj));
+	if (!obj)
+		return NULL;
+
+	bfd_init();
+	obj->bfd = bfd_openr(file, NULL);
+	if (!obj->bfd)
+		goto error;
+	obj->bfd->flags |= BFD_DECOMPRESS;
+
+	return obj;
+
+error:
+	debug_handle_destroy(obj);
+	return NULL;
+}
+
+/* Get the full path of process's executable, using the /proc fs. */
+static char *get_full_name(int pid)
+{
+	char mapname[PATH_MAX+1];
+	char fname[PATH_MAX+1];
+	int ret;
+
+	sprintf(fname, "/proc/%d/exe", pid);
+	ret = readlink(fname, mapname, PATH_MAX);
+	if (ret >= PATH_MAX || ret < 0)
+		return NULL;
+	mapname[ret] = 0;
+
+	return strdup(mapname);
+}
+
+/* Get or create a bfd debug context for an object file. */
+static struct debug_file *get_mapped_file(struct dbg_trace_context *dbg,
+					  char *fname,
+					  unsigned long long vmem_start)
+{
+	struct debug_file *file = dbg->files;
+
+	/* Search if the file is already added. */
+	while (file) {
+		if (!strcmp(fname, file->file_name) &&
+		    vmem_start && file->vmem_end == vmem_start)
+			return file;
+		file = file->next;
+	}
+
+	file = calloc(1, sizeof(*file));
+	if (!file)
+		return NULL;
+	file->file_name = strdup(fname);
+	if (!file->file_name)
+		goto error;
+	file->dbg = debug_handle_create(fname);
+	file->next = dbg->files;
+	dbg->files = file;
+	return file;
+
+error:
+	free(file->file_name);
+	debug_handle_destroy(file->dbg);
+	free(file);
+	return NULL;
+}
+
+/* Destroy a bfd debug context. */
+void dbg_trace_context_destroy(struct dbg_trace_context *dbg)
+{
+	struct debug_file *fdel;
+	struct debug_symbols *sdel;
+
+	while (dbg->sym) {
+		sdel = dbg->sym;
+		dbg->sym = dbg->sym->next;
+		free(sdel->symbol.name);
+		free(sdel->symbol.fname);
+		free(sdel);
+	}
+	while (dbg->files) {
+		fdel = dbg->files;
+		dbg->files = dbg->files->next;
+		debug_handle_destroy(fdel->dbg);
+		while (fdel->sym) {
+			sdel = fdel->sym;
+			fdel->sym = fdel->sym->next;
+			free(sdel->symbol.name);
+			free(sdel->symbol.fname);
+			free(sdel);
+		}
+		free(fdel);
+	}
+
+	free(dbg->fname);
+	dbg_trace_free_filemap(dbg->fmaps);
+	free(dbg);
+}
+
+/* Add an object file, mapped to specific memory of the process. */
+int dbg_trace_context_add_file(struct dbg_trace_context *dbg, char *file_name,
+			       unsigned long long vmem_start,
+			       unsigned long long vmem_end,
+			       unsigned long long pgoff)
+{
+	struct debug_file *file;
+
+	file = get_mapped_file(dbg, file_name, vmem_start);
+	if (!file)
+		return -1;
+	if (file->vmem_end == vmem_start) {
+		file->vmem_end = vmem_end;
+	} else {
+		file->vmem_start = vmem_start;
+		file->vmem_end = vmem_end;
+		if (file->dbg)
+			file->dbg->addr_offset = vmem_start - pgoff;
+	}
+
+	return 0;
+}
+
+/**
+ * dbg_trace_context_create_pid - create debug context for given PID
+ * @pid - ID of running process
+ * @libs - if true: inspect also all libraries, uased by the given process.
+ *
+ * Returns a pointer to allocated debug context, or NULL in case of an error.
+ */
+struct dbg_trace_context *dbg_trace_context_create_pid(int pid, bool libs)
+{
+	struct dbg_trace_context *dbg;
+	unsigned int i;
+
+	dbg = calloc(1, sizeof(*dbg));
+	if (!dbg)
+		return NULL;
+
+	dbg->pid = pid;
+	/* Get the full path of the process executable. */
+	dbg->fname = get_full_name(pid);
+	if (!dbg->fname) {
+		free(dbg);
+		return NULL;
+	}
+
+	/* Get the memory map of all libraries, linked to the process. */
+	dbg_trace_get_filemap(&dbg->fmaps, pid);
+
+	for (i = 0; i < dbg->fmaps->nr_lib_maps; i++) {
+		if (!libs && strcmp(dbg->fname, dbg->fmaps->lib_maps[i].lib_name))
+			continue;
+		/* Create a bfd debug object for each file. */
+		dbg_trace_context_add_file(dbg, dbg->fmaps->lib_maps[i].lib_name,
+					   dbg->fmaps->lib_maps[i].start,
+					   dbg->fmaps->lib_maps[i].end, 0);
+	}
+
+	return dbg;
+}
+
+/* Get the full path of a library. */
+static char *get_lib_full_path(char *libname)
+{
+	void *h = dlmopen(LM_ID_NEWLM, libname, RTLD_LAZY);
+	char dldir[PATH_MAX+1];
+	char *fname = NULL;
+	int ret;
+
+	if (!h)
+		return NULL;
+
+	ret = dlinfo(h, RTLD_DI_ORIGIN, dldir);
+	dlclose(h);
+
+	if (ret || asprintf(&fname, "%s/%s", dldir, libname) <= 0)
+		return NULL;
+
+	return fname;
+}
+
+/* Get the memory map of all libraries, linked to an executable file. */
+static int debug_obj_file_add_libs(struct dbg_trace_context *dbg,
+				   struct debug_file *file)
+{
+	char line[PATH_MAX];
+	char *libname;
+	char *trimmed;
+	char *fullname;
+	FILE *fp = NULL;
+	int ret = -1;
+
+	setenv("LD_TRACE_LOADED_OBJECTS", "1", 1);
+	fp = popen(file->file_name, "r");
+	if (!fp)
+		goto out;
+
+	while (fgets(line, sizeof(line), fp) != NULL) {
+		libname = strchr(line, ' ');
+		trimmed = line;
+		if (libname) {
+			*libname = '\0';
+			while (isspace(*trimmed))
+				trimmed++;
+			if (*trimmed != '/') {
+				fullname = get_lib_full_path(trimmed);
+				if (fullname) {
+					get_mapped_file(dbg, fullname, 0);
+					free(fullname);
+				}
+			} else {
+				get_mapped_file(dbg, trimmed, 0);
+			}
+		}
+	}
+
+out:
+	unsetenv("LD_TRACE_LOADED_OBJECTS");
+	if (fp)
+		pclose(fp);
+	return ret;
+}
+
+/**
+ * dbg_trace_context_create_file - create debug context for given executable file
+ * @fname - full path to an executable file
+ * @libs - if true: inspect also all libraries, used by the given file.
+ *
+ * Returns a pointer to allocated debug context, or NULL in case of an error.
+ */
+struct dbg_trace_context *dbg_trace_context_create_file(char *fname, bool libs)
+{
+	struct dbg_trace_context *dbg;
+	struct debug_file *file;
+
+	dbg = calloc(1, sizeof(*dbg));
+	if (!dbg)
+		return NULL;
+
+	dbg->fname = strdup(fname);
+	file = get_mapped_file(dbg, fname, 0);
+	if (!file)
+		goto error;
+	if (libs)
+		debug_obj_file_add_libs(dbg, file);
+
+#ifdef DEBUG_INTERNALS
+	printf("Created debug object for %s:\n\r", dbg->fname);
+	file = dbg->files;
+	while (file) {
+		printf("\t%s\n\r", file->file_name);
+		file = file->next;
+	}
+#endif
+	return dbg;
+
+error:
+	dbg_trace_context_destroy(dbg);
+	return NULL;
+}
+
+static void set_unknown(struct debug_symbols *sym, char *file)
+{
+	while (sym) {
+		if (!sym->symbol.fname)
+			sym->symbol.fname = strdup(file);
+		sym = sym->next;
+	}
+}
+
+/* Perform the requested symbols resolving, using the bfd library. */
+int dbg_trace_resolve_symbols(struct dbg_trace_context *obj)
+{
+	struct debug_file *file;
+
+	for (file = obj->files; file; file = file->next) {
+		if (!file->dbg) {
+			set_unknown(file->sym, file->file_name);
+			continue;
+		}
+		/* Resolve near VMA -> name. */
+		resolve_symbol_name(file->dbg, file->sym);
+		/* Resolve name -> exact VMA. */
+		resolve_symbol_vma(file->dbg, file->sym);
+		resolve_symbol_vma(file->dbg, obj->sym);
+	}
+
+	return 0;
+}
+
+/* Add VMA -> name resolving request. */
+static int add_resolve_vma2name(struct dbg_trace_context *obj,
+				unsigned long long vma, int cookie)
+{
+	struct debug_symbols *s = NULL;
+	struct debug_file *file;
+
+	file = obj->files;
+	while (file) {
+		/* Find the file, where the requested VMA is. */
+		if (vma >= file->vmem_start && vma <= file->vmem_end)
+			break;
+		file = file->next;
+	}
+
+	if (file) {
+		s = file->sym;
+		while (s) {
+			/* Check if the given VMA is already added for resolving. */
+			if (s->symbol.vma_near == vma)
+				break;
+			s = s->next;
+		}
+		if (!s) {
+			s = calloc(1, sizeof(*s));
+			if (!s)
+				return -1;
+			s->symbol.cookie = cookie;
+			s->symbol.vma_near = vma;
+			s->symbol.fname = strdup(file->file_name);
+			if (!s->symbol.fname)
+				goto error;
+			s->next = file->sym;
+			file->sym = s;
+			file->sym_count++;
+		}
+	}
+
+	if (s)
+		return 0;
+error:
+	if (s) {
+		free(s->symbol.fname);
+		free(s);
+	}
+	return -1;
+}
+
+/* Add name - VMA resolving request, The @name can have wildcards. */
+static int add_resolve_name2vma(struct dbg_trace_context *obj, char *name, int cookie)
+{
+	struct debug_symbols *s = NULL;
+
+	s = obj->sym;
+	while (s) {
+		/* Check if the given name is already added for resolving. */
+		if (s->symbol.name && !strcmp(name, s->symbol.name))
+			break;
+		s = s->next;
+	}
+	if (!s) {
+		s = calloc(1, sizeof(*s));
+		if (!s)
+			return -1;
+		s->symbol.cookie = cookie;
+		s->symbol.name = strdup(name);
+		if (!s->symbol.name)
+			goto error;
+		if (strchr(name, '*') || strchr(name, '?'))
+			s->match = MATH_WILDCARD;
+
+		s->next = obj->sym;
+		obj->sym = s;
+		obj->sym_count++;
+	}
+
+	return 0;
+
+error:
+	if (s) {
+		free(s->symbol.name);
+		free(s);
+	}
+	return -1;
+}
+
+/**
+ * dbg_trace_add_resolve_symbol - add new resolving request
+ * @obj - debug object context
+ * @vma - VMA->name resolving, if @vma is not 0
+ * @name - name-VMA resolving, if @name is not NULL
+ * @cookie - a cookie, attached to each successful resolving from this request
+ *
+ * Returns 0 if the request is added successfully, or -1 in case of an error.
+ */
+int dbg_trace_add_resolve_symbol(struct dbg_trace_context *obj,
+				 unsigned long long vma, char *name, int cookie)
+{
+	int ret = -1;
+
+	if (!obj)
+		return -1;
+
+	if (!name && vma) /* vma -> name resolving */
+		ret = add_resolve_vma2name(obj, vma, cookie);
+	else if (name) /* name -> vma resolving */
+		ret = add_resolve_name2vma(obj, name, cookie);
+
+	return ret;
+}
+
+static int walk_symbols(struct debug_symbols *sym,
+			int (*callback)(struct dbg_trace_symbols *, void *),
+			void *context)
+{
+	while (sym) {
+		if (callback(&sym->symbol, context))
+			return -1;
+		sym = sym->next;
+	}
+
+	return 0;
+}
+
+/**
+ * dbg_trace_walk_resolved_symbols - walk through all resolved symbols
+ * @obj - debug object context
+ * @callback - a callback hook, called for each resolved symbol.
+ *		If the callback returns non-zero, the walk stops.
+ * @context - a user specified context, passed to the callback
+ */
+void dbg_trace_walk_resolved_symbols(struct dbg_trace_context *obj,
+				     int (*callback)(struct dbg_trace_symbols *, void *),
+				     void *context)
+{
+	struct debug_file *file;
+
+	walk_symbols(obj->sym, callback, context);
+	file = obj->files;
+	while (file) {
+		walk_symbols(file->sym, callback, context);
+		file = file->next;
+	}
+}
+
+/**
+ * trace_debug_free_symbols - free array of debug symbols
+ * @symbols - array with debug symbols
+ * @count - count of the @symbols array
+ */
+void trace_debug_free_symbols(struct dbg_trace_symbols *symbols, int count)
+{
+	int i;
+
+	if (!symbols)
+		return;
+
+	for (i = 0; i < count; i++) {
+		free(symbols[i].name);
+		free(symbols[i].fname);
+	}
+	free(symbols);
+
+}
+
+#define _STRINGIFY(x) #x
+#define STRINGIFY(x) _STRINGIFY(x)
+/**
+ * dbg_trace_get_filemap - get a memory map of a process, using /proc fs
+ * @pid_maps - return: list of files, mapped into the process memory
+ * @pid - id of a process
+ *
+ * Returns 0 on success, -1 in case of an error.
+ */
+int dbg_trace_get_filemap(struct dbg_trace_pid_maps **pid_maps, int pid)
+{
+	struct dbg_trace_pid_maps *maps = *pid_maps;
+	struct dbg_trace_proc_addr_map *map;
+	unsigned long long begin, end;
+	struct dbg_trace_pid_maps *m;
+	char mapname[PATH_MAX+1];
+	char fname[PATH_MAX+1];
+	char buf[PATH_MAX+100];
+	unsigned int i;
+	FILE *f;
+	int ret;
+	int res;
+
+	sprintf(fname, "/proc/%d/exe", pid);
+	ret = readlink(fname, mapname, PATH_MAX);
+	if (ret >= PATH_MAX || ret < 0)
+		return -ENOENT;
+	mapname[ret] = 0;
+
+	sprintf(fname, "/proc/%d/maps", pid);
+	f = fopen(fname, "r");
+	if (!f)
+		return -ENOENT;
+
+	while (maps) {
+		if (pid == maps->pid)
+			break;
+		maps = maps->next;
+	}
+
+	ret = -ENOMEM;
+	if (!maps) {
+		maps = calloc(1, sizeof(*maps));
+		if (!maps)
+			goto out_fail;
+		maps->pid = pid;
+		maps->next = *pid_maps;
+		*pid_maps = maps;
+	} else {
+		for (i = 0; i < maps->nr_lib_maps; i++)
+			free(maps->lib_maps[i].lib_name);
+		free(maps->lib_maps);
+		maps->lib_maps = NULL;
+		maps->nr_lib_maps = 0;
+		free(maps->proc_name);
+	}
+
+	maps->proc_name = strdup(mapname);
+	if (!maps->proc_name)
+		goto out_fail;
+
+	while (fgets(buf, sizeof(buf), f)) {
+		mapname[0] = '\0';
+		res = sscanf(buf, "%llx-%llx %*s %*x %*s %*d %"STRINGIFY(PATH_MAX)"s",
+			     &begin, &end, mapname);
+		if (res == 3 && mapname[0] != '\0') {
+			map = realloc(maps->lib_maps,
+				      (maps->nr_lib_maps + 1) * sizeof(*map));
+			if (!map)
+				goto out_fail;
+			map[maps->nr_lib_maps].end = end;
+			map[maps->nr_lib_maps].start = begin;
+			map[maps->nr_lib_maps].lib_name = strdup(mapname);
+			if (!map[maps->nr_lib_maps].lib_name)
+				goto out_fail;
+			maps->lib_maps = map;
+			maps->nr_lib_maps++;
+		}
+	}
+
+	fclose(f);
+	return 0;
+
+out_fail:
+	fclose(f);
+	if (maps) {
+		for (i = 0; i < maps->nr_lib_maps; i++)
+			free(maps->lib_maps[i].lib_name);
+		if (*pid_maps != maps) {
+			m = *pid_maps;
+			while (m) {
+				if (m->next == maps) {
+					m->next = maps->next;
+					break;
+				}
+				m = m->next;
+			}
+		} else
+			*pid_maps = maps->next;
+		free(maps->lib_maps);
+		maps->lib_maps = NULL;
+		maps->nr_lib_maps = 0;
+		free(maps->proc_name);
+		maps->proc_name = NULL;
+		free(maps);
+	}
+	return ret;
+}
+
+static void procmap_free(struct dbg_trace_pid_maps *maps)
+{
+	unsigned int i;
+
+	if (!maps)
+		return;
+	if (maps->lib_maps) {
+		for (i = 0; i < maps->nr_lib_maps; i++)
+			free(maps->lib_maps[i].lib_name);
+		free(maps->lib_maps);
+	}
+	free(maps->proc_name);
+	free(maps);
+}
+
+/**
+ * dbg_trace_free_filemap - Free list of files, associated with given process
+ * @maps - list of files, returned by dbg_trace_get_filemap()
+ */
+void dbg_trace_free_filemap(struct dbg_trace_pid_maps *maps)
+{
+	struct dbg_trace_pid_maps *del;
+
+	while (maps) {
+		del = maps;
+		maps = maps->next;
+		procmap_free(del);
+	}
+}
diff --git a/src/trace-obj-debug.h b/src/trace-obj-debug.h
new file mode 100644
index 0000000..19091ad
--- /dev/null
+++ b/src/trace-obj-debug.h
@@ -0,0 +1,54 @@
+/* SPDX-License-Identifier: LGPL-2.1 */
+
+/*
+ * Copyright 2022 VMware Inc, Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
+ */
+
+#ifndef _TC_TRACE_DEBUG_UTILS_
+#define _TC_TRACE_DEBUG_UTILS_
+
+/* --- Debug symbols--- */
+struct dbg_trace_pid_maps {
+	struct dbg_trace_pid_maps	*next;
+	struct dbg_trace_proc_addr_map	*lib_maps;
+	unsigned int			nr_lib_maps;
+	char				*proc_name;
+	int				pid;
+};
+int dbg_trace_get_filemap(struct dbg_trace_pid_maps **file_maps, int pid);
+void dbg_trace_free_filemap(struct dbg_trace_pid_maps *maps);
+
+struct dbg_trace_symbols {
+	char *name;			/* symbol's name */
+	char *fname;			/* symbol's file */
+	int cookie;
+	unsigned long long vma_start;	/* symbol's start VMA */
+	unsigned long long vma_near;	/* symbol's requested VMA */
+	unsigned long long foffset;	/* symbol's offset in the binary file*/
+};
+
+struct dbg_trace_proc_addr_map {
+	unsigned long long	start;
+	unsigned long long	end;
+	char			*lib_name;
+};
+
+struct dbg_trace_context;
+struct dbg_trace_context *dbg_trace_context_create_file(char *file, bool libs);
+struct dbg_trace_context *dbg_trace_context_create_pid(int pid, bool libs);
+
+void dbg_trace_context_destroy(struct dbg_trace_context *debug);
+int dbg_trace_context_add_file(struct dbg_trace_context *dbg, char *file_name,
+			       unsigned long long vmem_start,
+			       unsigned long long vmem_end,
+			       unsigned long long pgoff);
+
+int dbg_trace_resolve_symbols(struct dbg_trace_context *obj);
+int dbg_trace_add_resolve_symbol(struct dbg_trace_context *obj,
+				 unsigned long long vma, char *name, int cookie);
+
+void dbg_trace_walk_resolved_symbols(struct dbg_trace_context *obj,
+				     int (*callback)(struct dbg_trace_symbols *, void *),
+				     void *context);
+
+#endif /* _TC_TRACE_DEBUG_UTILS_ */
-- 
2.35.1

[RFC PATCH v2 2/4] trace-cruncher: ftrace uprobe raw API

[Tzvetomir Stoyanov (VMware)]

Trace-cruncher low level wrappers for tracefs library APIs for
uprobe and uretprobe allocation.

Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
 src/ftracepy-utils.c | 48 ++++++++++++++++++++++++++++++++++++++++++++
 src/ftracepy-utils.h |  4 ++++
 src/ftracepy.c       | 10 +++++++++
 3 files changed, 62 insertions(+)

diff --git a/src/ftracepy-utils.c b/src/ftracepy-utils.c
index e8411ae..b39459b 100644
--- a/src/ftracepy-utils.c
+++ b/src/ftracepy-utils.c
@@ -2449,6 +2449,54 @@ PyObject *PyFtrace_eprobe(PyObject *self, PyObject *args, PyObject *kwargs)
 	return py_dyn;
 }
 
+static PyObject *alloc_uprobe(PyObject *self, PyObject *args, PyObject *kwargs, bool pret)
+{
+	static char *kwlist[] = {"event", "file", "offset", "fetch_args", NULL};
+	const char *event, *file, *fetchargs = NULL;
+	unsigned long long offset;
+	struct tracefs_dynevent *uprobe;
+	PyObject *py_dyn;
+
+	if (!PyArg_ParseTupleAndKeywords(args,
+					 kwargs,
+					 "ssK|s",
+					 kwlist,
+					 &event,
+					 &file,
+					 &offset,
+					 &fetchargs)) {
+		return NULL;
+	}
+
+	if (pret)
+		uprobe = tracefs_uretprobe_alloc(TC_SYS, event, file, offset, fetchargs);
+	else
+		uprobe = tracefs_uprobe_alloc(TC_SYS, event, file, offset, fetchargs);
+	if (!uprobe) {
+		MEM_ERROR;
+		return NULL;
+	}
+
+	py_dyn = PyDynevent_New(uprobe);
+	/*
+	 * Here we only allocated and initializes a dynamic event object.
+	 * However, no dynamic event is added to the system yet. Hence,
+	 * there is no need to 'destroy' this event at exit.
+	 */
+	set_destroy_flag(py_dyn, false);
+	return py_dyn;
+}
+
+PyObject *PyFtrace_uprobe(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+	return alloc_uprobe(self, args, kwargs, false);
+}
+
+PyObject *PyFtrace_uretprobe(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+	return alloc_uprobe(self, args, kwargs, true);
+}
+
 static PyObject *set_filter(PyObject *args, PyObject *kwargs,
 			    struct tep_handle *tep,
 			    struct tep_event *event)
diff --git a/src/ftracepy-utils.h b/src/ftracepy-utils.h
index 9491b18..e6fab69 100644
--- a/src/ftracepy-utils.h
+++ b/src/ftracepy-utils.h
@@ -257,6 +257,10 @@ PyObject *PyFtrace_kretprobe(PyObject *self, PyObject *args, PyObject *kwargs);
 
 PyObject *PyFtrace_eprobe(PyObject *self, PyObject *args, PyObject *kwargs);
 
+PyObject *PyFtrace_uprobe(PyObject *self, PyObject *args, PyObject *kwargs);
+
+PyObject *PyFtrace_uretprobe(PyObject *self, PyObject *args, PyObject *kwargs);
+
 PyObject *PyFtrace_hist(PyObject *self, PyObject *args,
 					PyObject *kwargs);
 
diff --git a/src/ftracepy.c b/src/ftracepy.c
index 574bf38..681d641 100644
--- a/src/ftracepy.c
+++ b/src/ftracepy.c
@@ -481,6 +481,16 @@ static PyMethodDef ftracepy_methods[] = {
 	 METH_VARARGS | METH_KEYWORDS,
 	 "Define an eprobe."
 	},
+	{"uprobe",
+	 (PyCFunction) PyFtrace_uprobe,
+	 METH_VARARGS | METH_KEYWORDS,
+	 "Define a uprobe."
+	},
+	{"uretprobe",
+	 (PyCFunction) PyFtrace_uretprobe,
+	 METH_VARARGS | METH_KEYWORDS,
+	 "Define a uretprobe."
+	},
 	{"hist",
 	 (PyCFunction) PyFtrace_hist,
 	 METH_VARARGS | METH_KEYWORDS,
-- 
2.35.1

[RFC PATCH v2 3/4] trace-cruncher: High level wrappers for ftrace uprobes

[Tzvetomir Stoyanov (VMware)]

Using uprobes requires finding the offset of a user function within the
binary file, where this functions is compiled. This is not a trivial
task, especially in the cases when a bunch of uprobes to user functions
should be added.
A high level trace-cruncher API allows adding multiple user functions as
uprobes or uretprobes. It supports wildcards for function names and
adding uprobes for library functions, used by the applications.

Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
 setup.py             |   4 +-
 src/ftracepy-utils.c | 628 +++++++++++++++++++++++++++++++++++++++++++
 src/ftracepy-utils.h |  17 ++
 src/ftracepy.c       |  35 +++
 4 files changed, 682 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 58561cf..464d3d2 100644
--- a/setup.py
+++ b/setup.py
@@ -71,8 +71,8 @@ def extension(name, sources, libraries):
 
 def main():
     module_ft = extension(name='tracecruncher.ftracepy',
-                          sources=['src/ftracepy.c', 'src/ftracepy-utils.c'],
-                          libraries=['traceevent', 'tracefs'])
+                          sources=['src/ftracepy.c', 'src/ftracepy-utils.c', 'src/trace-obj-debug.c'],
+                          libraries=['traceevent', 'tracefs', 'bfd'])
 
     cythonize('src/npdatawrapper.pyx', language_level = 3)
     module_data = extension(name='tracecruncher.npdatawrapper',
diff --git a/src/ftracepy-utils.c b/src/ftracepy-utils.c
index b39459b..d420dce 100644
--- a/src/ftracepy-utils.c
+++ b/src/ftracepy-utils.c
@@ -15,15 +15,46 @@
 #include <sys/utsname.h>
 #include <sys/wait.h>
 #include <signal.h>
+#include <semaphore.h>
 #include <time.h>
 
 // trace-cruncher
 #include "ftracepy-utils.h"
+#include "trace-obj-debug.h"
 
 PyObject *TFS_ERROR;
 PyObject *TEP_ERROR;
 PyObject *TRACECRUNCHER_ERROR;
 
+#define UPROBES_SYSTEM "tc_uprobes"
+
+#define FTRACE_UPROBE		0x1
+#define FTRACE_URETPROBE	0x2
+
+struct fprobes_list {
+	int size;
+	int count;
+	void **data;
+};
+
+struct utrace_func {
+	int type;
+	char *func_name;
+	char *func_args;
+};
+
+struct py_utrace_context {
+	pid_t pid;
+	char *fname;
+	char **argv;
+	char *usystem;
+	uint32_t trace_time; /* in msec */
+	struct fprobes_list fretprobes;
+	struct fprobes_list ufuncs;
+	struct fprobes_list uevents;
+	struct dbg_trace_context *dbg;
+};
+
 static char *kernel_version()
 {
 	struct utsname uts;
@@ -3508,3 +3539,600 @@ void PyFtrace_at_exit(void)
 	if (seq.buffer)
 		trace_seq_destroy(&seq);
 }
+
+#define INIT_SIZE	100
+static int utrace_list_add(struct fprobes_list *list, void *data)
+{
+	void **tmp;
+	int size;
+
+	if (list->size <= list->count) {
+		if (!list->size)
+			size = INIT_SIZE;
+		else
+			size = 2 * list->size;
+		tmp = realloc(list->data, size * sizeof(void *));
+		if (!tmp)
+			return -1;
+		list->data = tmp;
+		list->size = size;
+	}
+
+	list->data[list->count] = data;
+	list->count++;
+	return list->count - 1;
+}
+
+void py_utrace_free(struct py_utrace_context *utrace)
+{
+	struct utrace_func *f;
+	int i;
+
+	if (!utrace)
+		return;
+	if (utrace->dbg)
+		dbg_trace_context_destroy(utrace->dbg);
+
+	for (i = 0; i < utrace->ufuncs.count; i++) {
+		f = utrace->ufuncs.data[i];
+		free(f->func_name);
+		free(f);
+	}
+	free(utrace->ufuncs.data);
+	if (utrace->argv) {
+		i = 0;
+		while (utrace->argv[i])
+			free(utrace->argv[i++]);
+		free(utrace->argv);
+	}
+
+	for (i = 0; i < utrace->uevents.count; i++)
+		tracefs_dynevent_free(utrace->uevents.data[i]);
+	free(utrace->uevents.data);
+
+	free(utrace->fname);
+	free(utrace->usystem);
+	free(utrace);
+}
+
+/*
+ * All strings, used as ftrace system or event name must contain only
+ * alphabetic characters, digits or underscores.
+ */
+static void fname_unify(char *fname)
+{
+	int i;
+
+	for (i = 0; fname[i]; i++)
+		if (!isalnum(fname[i]) && fname[i] != '_')
+			fname[i] = '_';
+}
+
+int py_utrace_destroy(struct py_utrace_context *utrace)
+{
+	int i;
+
+	for (i = 0; i < utrace->uevents.count; i++)
+		tracefs_dynevent_destroy(utrace->uevents.data[i], true);
+
+	return 0;
+}
+
+static struct py_utrace_context *utrace_new(pid_t pid, char *fname, char **argv, bool libs)
+{
+	struct py_utrace_context *utrace;
+	char *file;
+
+	utrace = calloc(1, sizeof(*utrace));
+	if (!utrace)
+		return NULL;
+
+	if (fname) {
+		utrace->argv = argv;
+		utrace->dbg = dbg_trace_context_create_file(fname, libs);
+		if (!utrace->dbg)
+			goto error;
+		utrace->fname = strdup(fname);
+		if (!utrace->fname)
+			goto error;
+		file = strrchr(fname, '/');
+		if (file)
+			file++;
+		if (!file || *file == '\0')
+			file = fname;
+		if (asprintf(&utrace->usystem, "%s_%s", UPROBES_SYSTEM, file) <= 0)
+			goto error;
+	} else {
+		utrace->pid = pid;
+		utrace->dbg = dbg_trace_context_create_pid(pid, libs);
+		if (!utrace->dbg)
+			goto error;
+		if (asprintf(&utrace->usystem, "%s_%d", UPROBES_SYSTEM, pid) <= 0)
+			goto error;
+	}
+
+	fname_unify(utrace->usystem);
+	return utrace;
+
+error:
+	py_utrace_free(utrace);
+	return NULL;
+}
+
+static int py_utrace_add_func(struct py_utrace_context *utrace, char *func, int type)
+{
+	struct utrace_func *p;
+	int ret;
+	int i;
+
+	for (i = 0; i < utrace->ufuncs.count; i++) {
+		p = utrace->ufuncs.data[i];
+		if (!strcmp(p->func_name, func)) {
+			p->type |= type;
+			return 0;
+		}
+	}
+
+	p = calloc(1, sizeof(*p));
+	if (!p)
+		return -1;
+	p->func_name = strdup(func);
+	if (!p->func_name)
+		goto error;
+	p->type = type;
+
+	ret = utrace_list_add(&utrace->ufuncs, p);
+	if (ret < 0)
+		goto error;
+
+	if (dbg_trace_add_resolve_symbol(utrace->dbg, 0, func, ret))
+		goto error;
+
+	return 0;
+
+error:
+	free(p->func_name);
+	free(p);
+	return -1;
+}
+
+PyObject *PyUserTrace_add_function(PyUserTrace *self, PyObject *args,
+				   PyObject *kwargs)
+{
+	struct py_utrace_context *utrace = self->ptrObj;
+	static char *kwlist[] = {"fname", NULL};
+	char *fname;
+
+	if (!PyArg_ParseTupleAndKeywords(args,
+					 kwargs,
+					 "s",
+					 kwlist,
+					 &fname)) {
+		return NULL;
+	}
+
+	if (py_utrace_add_func(utrace, fname, FTRACE_UPROBE) < 0) {
+		MEM_ERROR
+		return NULL;
+	}
+
+	Py_RETURN_NONE;
+}
+
+PyObject *PyUserTrace_add_ret_function(PyUserTrace *self, PyObject *args,
+				       PyObject *kwargs)
+{
+	struct py_utrace_context *utrace = self->ptrObj;
+	static char *kwlist[] = {"fname", NULL};
+	char *fname;
+
+	if (!PyArg_ParseTupleAndKeywords(args,
+					 kwargs,
+					 "s",
+					 kwlist,
+					 &fname)) {
+		return NULL;
+	}
+
+	if (py_utrace_add_func(utrace, fname, FTRACE_URETPROBE) < 0) {
+		MEM_ERROR
+		return NULL;
+	}
+
+	Py_RETURN_NONE;
+}
+
+/*
+ * max event name is 64 bytes, hard coded in the kernel.
+ * it can consists only of alphabetic characters, digits or underscores
+ */
+#define FILENAME_TRUNCATE	10
+#define FUNCAME_TRUNCATE	50
+static char *uprobe_event_name(char *file, char *func, int type)
+{
+	char *event = NULL;
+	char *fname;
+
+	fname = strrchr(file, '/');
+	if (fname)
+		fname++;
+	if (!fname || *fname == '\0')
+		fname = file;
+
+	asprintf(&event, "%s%.*s_%.*s",
+		 type == FTRACE_URETPROBE ? "r_":"",
+		 FILENAME_TRUNCATE, fname, FUNCAME_TRUNCATE, func);
+	if (event)
+		fname_unify(event);
+
+	return event;
+}
+
+/*
+ * Create uprobe based on function name,
+ * file name and function offset within the file
+ */
+static int utrace_event_create(struct py_utrace_context *utrace,
+			       struct dbg_trace_symbols *sym, char *fecthargs,
+			       int type)
+{
+	struct tracefs_dynevent *uevent = NULL;
+	char *rname;
+
+	/* Generate uprobe event name, according to ftrace name requirements */
+	rname = uprobe_event_name(sym->fname, sym->name, type);
+	if (!rname)
+		return -1;
+
+	if (type == FTRACE_URETPROBE)
+		uevent = tracefs_uretprobe_alloc(utrace->usystem, rname,
+						 sym->fname, sym->foffset, fecthargs);
+	else
+		uevent = tracefs_uprobe_alloc(utrace->usystem, rname,
+					      sym->fname, sym->foffset, fecthargs);
+
+	free(rname);
+	if (!uevent)
+		return -1;
+
+	if (tracefs_dynevent_create(uevent)) {
+		tracefs_dynevent_free(uevent);
+		return -1;
+	}
+
+	utrace_list_add(&utrace->uevents, uevent);
+	return 0;
+}
+
+/* callback, called on each resolved function */
+static int symblos_walk(struct dbg_trace_symbols *sym, void *context)
+{
+	struct py_utrace_context *utrace = context;
+	struct utrace_func *ufunc;
+
+	if (!sym->name || !sym->fname || !sym->foffset ||
+	    sym->cookie < 0 || sym->cookie >= utrace->ufuncs.count)
+		return 0;
+
+	ufunc = utrace->ufuncs.data[sym->cookie];
+
+	if (ufunc->type & FTRACE_UPROBE)
+		utrace_event_create(utrace, sym, ufunc->func_args, FTRACE_UPROBE);
+
+	if (ufunc->type & FTRACE_URETPROBE)
+		utrace_event_create(utrace, sym, ufunc->func_args, FTRACE_URETPROBE);
+
+	return 0;
+}
+
+static void py_utrace_generate_uprobes(struct py_utrace_context *utrace)
+{
+	/* Find the exact name and file offset of each user function that should be traced */
+	dbg_trace_resolve_symbols(utrace->dbg);
+	dbg_trace_walk_resolved_symbols(utrace->dbg, symblos_walk, utrace);
+}
+
+static int start_trace(struct py_utrace_context *utrace, struct tracefs_instance *instance)
+{
+	PyObject *pPid = PyLong_FromLong(utrace->pid);
+	bool ret;
+
+	if (!pPid)
+		return -1;
+
+	/* Filter the trace only on desired pid(s) */
+	ret = hook2pid(instance, PyLong_FromLong(utrace->pid), true);
+	Py_DECREF(pPid);
+	if (!ret) {
+		PyErr_SetString(TRACECRUNCHER_ERROR,
+				"Failed to set trace filter");
+		return -1;
+	}
+
+	/* Enable uprobes in the system */
+	if (tracefs_event_enable(instance, utrace->usystem, NULL)) {
+		PyErr_SetString(TRACECRUNCHER_ERROR,
+				"Failed to enable trace events");
+		return -1;
+	}
+
+	return 0;
+}
+
+#define PERF_EXEC_SYNC	"/TC_PERF_SYNC_XXXXXX"
+static int utrace_exec_cmd(struct py_utrace_context *utrace, struct tracefs_instance *instance)
+{
+	char *envp[] = {NULL};
+	char sname[strlen(PERF_EXEC_SYNC) + 1];
+	sem_t *sem;
+	pid_t pid;
+	int ret;
+
+	strcpy(sname, PERF_EXEC_SYNC);
+	mktemp(sname);
+	sem = sem_open(sname, O_CREAT | O_EXCL, 0644, 0);
+	sem_unlink(sname);
+
+	pid = fork();
+	if (pid < 0) {
+		PyErr_SetString(TRACECRUNCHER_ERROR, "Failed to fork");
+		return -1;
+	}
+	if (pid == 0) {
+		sem_wait(sem);
+		execvpe(utrace->fname, utrace->argv, envp);
+	} else {
+		utrace->pid = pid;
+		start_trace(utrace, instance);
+		sem_post(sem);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int py_utrace_enable(struct py_utrace_context *utrace, struct tracefs_instance *instance)
+{
+	/* If uprobes on desired user functions are not yet generated, do it now */
+	if (!utrace->uevents.count)
+		py_utrace_generate_uprobes(utrace);
+
+	/* No functions are found in the given program / pid */
+	if (!utrace->uevents.count) {
+		PyErr_SetString(TRACECRUNCHER_ERROR,
+				"Cannot find requested user functions");
+		return -1;
+	}
+
+	if (utrace->fname)
+		utrace_exec_cmd(utrace, instance);
+	else
+		start_trace(utrace, instance);
+
+	return 0;
+}
+
+static int py_utrace_disable(struct py_utrace_context *utrace, struct tracefs_instance *instance)
+{
+	/* Disable uprobes in the system */
+	if (tracefs_event_disable(instance, utrace->usystem, NULL)) {
+		PyErr_SetString(TRACECRUNCHER_ERROR,
+				"Failed to disable trace events");
+		return -1;
+	}
+
+	return 0;
+}
+
+static bool tracing_run;
+
+static void tracing_stop(int sig)
+{
+	tracing_run = false;
+}
+
+static void tracing_timer(int sig, siginfo_t *si, void *uc)
+{
+	tracing_run = false;
+}
+
+#define PID_WAIT_CHECK_USEC	500000
+#define TIMER_SEC_NANO		1000000000LL
+static int utrace_wait_pid(struct py_utrace_context *utrace)
+{
+	struct itimerspec tperiod = {0};
+	struct sigaction saction = {0};
+	struct sigevent stime = {0};
+	timer_t timer_id;
+
+	if (utrace->pid == 0)
+		return -1;
+
+	tracing_run = true;
+	signal(SIGINT, tracing_stop);
+
+	if (utrace->trace_time) {
+		stime.sigev_notify = SIGEV_SIGNAL;
+		stime.sigev_signo = SIGRTMIN;
+		if (timer_create(CLOCK_MONOTONIC, &stime, &timer_id))
+			return -1;
+		saction.sa_flags = SA_SIGINFO;
+		saction.sa_sigaction = tracing_timer;
+		sigemptyset(&saction.sa_mask);
+		if (sigaction(SIGRTMIN, &saction, NULL)) {
+			timer_delete(timer_id);
+			return -1;
+		}
+		/* covert trace_time from msec to sec, nsec */
+		tperiod.it_value.tv_nsec = ((unsigned long long)utrace->trace_time * 1000000LL);
+		if (tperiod.it_value.tv_nsec >= TIMER_SEC_NANO) {
+			tperiod.it_value.tv_sec = tperiod.it_value.tv_nsec / TIMER_SEC_NANO;
+			tperiod.it_value.tv_nsec %= TIMER_SEC_NANO;
+		}
+		if (timer_settime(timer_id, 0, &tperiod, NULL))
+			return -1;
+	}
+
+	do {
+		if (utrace->fname) { /* wait for a child */
+			if (waitpid(utrace->pid, NULL, WNOHANG) == (int)utrace->pid) {
+				utrace->pid = 0;
+				tracing_run = false;
+			}
+		} else { /* not a child, check if still exist */
+			if (kill(utrace->pid, 0) == -1 && errno == ESRCH) {
+				utrace->pid = 0;
+				tracing_run = false;
+			}
+		}
+		usleep(PID_WAIT_CHECK_USEC);
+	} while (tracing_run);
+
+	if (utrace->trace_time)
+		timer_delete(timer_id);
+
+	signal(SIGINT, SIG_DFL);
+
+	return 0;
+}
+
+PyObject *PyUserTrace_enable(PyUserTrace *self, PyObject *args, PyObject *kwargs)
+{
+	struct py_utrace_context *utrace = self->ptrObj;
+	static char *kwlist[] = {"instance", "wait", "time", NULL};
+	struct tracefs_instance *instance = NULL;
+	PyObject *py_inst = NULL;
+	int wait = false;
+	int ret;
+
+	if (!utrace) {
+		PyErr_SetString(TRACECRUNCHER_ERROR,
+				"Failed to get utrace context");
+		return NULL;
+	}
+
+	if (!PyArg_ParseTupleAndKeywords(args,
+					 kwargs,
+					 "|Ops",
+					 kwlist,
+					 &py_inst,
+					 &wait,
+					 &utrace->trace_time)) {
+		PyErr_SetString(TRACECRUNCHER_ERROR,
+				"Failed to parse input arguments");
+		return NULL;
+	}
+
+	if (!get_optional_instance(py_inst, &instance))
+		return NULL;
+
+	ret = py_utrace_enable(utrace, instance);
+	if (ret)
+		return NULL;
+
+	if (wait) {
+		utrace_wait_pid(utrace);
+		py_utrace_disable(utrace, instance);
+	}
+
+	Py_RETURN_NONE;
+}
+
+PyObject *PyUserTrace_disable(PyUserTrace *self, PyObject *args, PyObject *kwargs)
+{
+	struct py_utrace_context *utrace = self->ptrObj;
+	static char *kwlist[] = {"instance", NULL};
+	struct tracefs_instance *instance = NULL;
+	PyObject *py_inst = NULL;
+	int ret;
+
+	if (!utrace) {
+		PyErr_SetString(TRACECRUNCHER_ERROR,
+				"Failed to get utrace context");
+		return NULL;
+	}
+
+	if (!PyArg_ParseTupleAndKeywords(args,
+					 kwargs,
+					 "|O",
+					 kwlist,
+					 &py_inst)) {
+		PyErr_SetString(TRACECRUNCHER_ERROR,
+				"Failed to parse input arguments");
+		return NULL;
+	}
+
+	if (!get_optional_instance(py_inst, &instance))
+		return NULL;
+
+	ret = py_utrace_disable(utrace, instance);
+
+	if (ret)
+		return NULL;
+
+	Py_RETURN_NONE;
+}
+
+PyObject *PyFtrace_utrace(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+	static char *kwlist[] = {"pid", "name", "follow_libs", "argv", NULL};
+	struct py_utrace_context *utrace;
+	char **argv = NULL;
+	long long pid = 0;
+	char *comm = NULL;
+	int libs = 0;
+	PyObject *py_utrace, *py_arg, *py_args = NULL;
+	int i, argc;
+
+	if (!PyArg_ParseTupleAndKeywords(args,
+					 kwargs,
+					 "|KspO",
+					 kwlist,
+					 &pid,
+					 &comm,
+					 &libs,
+					 &py_args)) {
+		return NULL;
+	}
+
+	if (pid == 0 && !comm) {
+		PyErr_Format(TFS_ERROR,
+			     "Process ID or program name should be specified");
+		return NULL;
+	}
+
+	if (comm && py_args) {
+		if (!PyList_CheckExact(py_args)) {
+			PyErr_SetString(TFS_ERROR, "Failed to parse argv list");
+			return NULL;
+		}
+		argc = PyList_Size(py_args);
+		argv = calloc(argc + 1, sizeof(char *));
+		for (i = 0; i < argc; i++) {
+			py_arg = PyList_GetItem(py_args, i);
+			if (!PyUnicode_Check(py_arg))
+				continue;
+			argv[i] = strdup(PyUnicode_DATA(py_arg));
+			if (!argv[i])
+				goto error;
+		}
+		argv[i] = NULL;
+	}
+
+	utrace = utrace_new(pid, comm, argv, libs);
+	if (!utrace) {
+		MEM_ERROR;
+		goto error;
+	}
+	py_utrace = PyUserTrace_New(utrace);
+
+	return py_utrace;
+
+error:
+	if (argv) {
+		for (i = 0; i < argc; i++)
+			free(argv[i]);
+		free(argv);
+	}
+	return NULL;
+}
diff --git a/src/ftracepy-utils.h b/src/ftracepy-utils.h
index e6fab69..0b5000b 100644
--- a/src/ftracepy-utils.h
+++ b/src/ftracepy-utils.h
@@ -34,6 +34,21 @@ C_OBJECT_WRAPPER_DECLARE(tracefs_synth, PySynthEvent)
 
 PyObject *PyTepRecord_time(PyTepRecord* self);
 
+struct py_utrace_context;
+void py_utrace_free(struct py_utrace_context *utrace);
+int py_utrace_destroy(struct py_utrace_context *utrace);
+C_OBJECT_WRAPPER_DECLARE(py_utrace_context, PyUserTrace);
+
+PyObject *PyUserTrace_add_function(PyUserTrace *self, PyObject *args,
+				   PyObject *kwargs);
+
+PyObject *PyUserTrace_add_ret_function(PyUserTrace *self, PyObject *args,
+				       PyObject *kwargs);
+
+PyObject *PyUserTrace_enable(PyUserTrace *self, PyObject *args, PyObject *kwargs);
+
+PyObject *PyUserTrace_disable(PyUserTrace *self, PyObject *args, PyObject *kwargs);
+
 PyObject *PyTepRecord_cpu(PyTepRecord* self);
 
 PyObject *PyTepEvent_name(PyTepEvent* self);
@@ -270,6 +285,8 @@ PyObject *PyFtrace_synth(PyObject *self, PyObject *args,
 PyObject *PyFtrace_set_ftrace_loglevel(PyObject *self, PyObject *args,
 						       PyObject *kwargs);
 
+PyObject *PyFtrace_utrace(PyObject *self, PyObject *args, PyObject *kwargs);
+
 PyObject *PyFtrace_trace_process(PyObject *self, PyObject *args,
 						 PyObject *kwargs);
 
diff --git a/src/ftracepy.c b/src/ftracepy.c
index 681d641..6ca9df0 100644
--- a/src/ftracepy.c
+++ b/src/ftracepy.c
@@ -315,6 +315,32 @@ C_OBJECT_WRAPPER(tracefs_synth, PySynthEvent,
 		 tracefs_synth_destroy,
 		 tracefs_synth_free)
 
+static PyMethodDef PyUserTrace_methods[] = {
+	{"add_function",
+	 (PyCFunction) PyUserTrace_add_function,
+	 METH_VARARGS | METH_KEYWORDS,
+	 "Add tracepoint on user function."
+	},
+	{"add_ret_function",
+	 (PyCFunction) PyUserTrace_add_ret_function,
+	 METH_VARARGS | METH_KEYWORDS,
+	 "Add tracepoint on user function return."
+	},
+	{"enable",
+	 (PyCFunction) PyUserTrace_enable,
+	 METH_VARARGS | METH_KEYWORDS,
+	 "Enable tracing of the configured user tracepoints."
+	},
+	{"disable",
+	 (PyCFunction) PyUserTrace_disable,
+	 METH_VARARGS | METH_KEYWORDS,
+	 "Disable tracing of the configured user tracepoints."
+	},
+	{NULL, NULL, 0, NULL}
+};
+C_OBJECT_WRAPPER(py_utrace_context, PyUserTrace,
+		 py_utrace_destroy, py_utrace_free)
+
 static PyMethodDef ftracepy_methods[] = {
 	{"dir",
 	 (PyCFunction) PyFtrace_dir,
@@ -501,6 +527,11 @@ static PyMethodDef ftracepy_methods[] = {
 	 METH_VARARGS | METH_KEYWORDS,
 	 "Define a synthetic event."
 	},
+	{"user_trace",
+	 (PyCFunction) PyFtrace_utrace,
+	 METH_VARARGS | METH_KEYWORDS,
+	 "Create a context for tracing a user process using uprobes"
+	},
 	{"set_ftrace_loglevel",
 	 (PyCFunction) PyFtrace_set_ftrace_loglevel,
 	 METH_VARARGS | METH_KEYWORDS,
@@ -575,6 +606,9 @@ PyMODINIT_FUNC PyInit_ftracepy(void)
 	if (!PySynthEventTypeInit())
 		return NULL;
 
+	if (!PyUserTraceTypeInit())
+		return NULL;
+
 	TFS_ERROR = PyErr_NewException("tracecruncher.ftracepy.tfs_error",
 				       NULL, NULL);
 
@@ -593,6 +627,7 @@ PyMODINIT_FUNC PyInit_ftracepy(void)
 	PyModule_AddObject(module, "tracefs_dynevent", (PyObject *) &PyDyneventType);
 	PyModule_AddObject(module, "tracefs_hist", (PyObject *) &PyTraceHistType);
 	PyModule_AddObject(module, "tracefs_synth", (PyObject *) &PySynthEventType);
+	PyModule_AddObject(module, "py_utrace_context", (PyObject *) &PyUserTraceType);
 
 	PyModule_AddObject(module, "tfs_error", TFS_ERROR);
 	PyModule_AddObject(module, "tep_error", TEP_ERROR);
-- 
2.35.1

[RFC PATCH v2 4/4] trace-cruncher: Example script for uprobes high level API

[Tzvetomir Stoyanov (VMware)]

Proposed example illustrates how to use uprobes high level API to trace
all functions for user program. It can be attached to already running
program, or run the program and trace its execution.

Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
---
 examples/user_trace.py | 43 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100755 examples/user_trace.py

diff --git a/examples/user_trace.py b/examples/user_trace.py
new file mode 100755
index 0000000..640ace8
--- /dev/null
+++ b/examples/user_trace.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+"""
+SPDX-License-Identifier: CC-BY-4.0
+
+Copyright 2022 VMware Inc, Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
+"""
+
+import sys
+import shutil
+
+import tracecruncher.ftracepy as ft
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print('Usage: ', sys.argv[0], ' [PROCESS]')
+        sys.exit(1)
+
+    # Create new Ftrace instance to work in. The tracing in this new instance
+    # is not going to be enabled yet.
+    inst = ft.create_instance(tracing_on=False)
+
+    # Create a user tracing context for given process, exclude the libraries
+    if sys.argv[1].isdigit():
+        utrace = ft.user_trace(pid=int(sys.argv[1]), follow_libs=False)
+    else:
+        file = shutil.which(sys.argv[1])
+        if file is None:
+            print('Cannot find ', sys.argv[1], ' in the system')
+            sys.exit(1)
+        utrace = ft.user_trace(name=file, argv=sys.argv[1:], follow_libs=False)
+
+    # Trace execution of all available functions in the given process
+    utrace.add_function(fname = "*")
+
+    # Add trace points on functions return as well
+    utrace.add_ret_function(fname = "*")
+
+    # Start tracing in an instance
+    utrace.enable(instance = inst, wait=False)
+
+    # Read the trace buffer during the trace until ctrl-c is pressed
+    ft.read_trace(instance=inst)
\ No newline at end of file
-- 
2.35.1

Re: [RFC PATCH v2 1/4] trace-cruncher: Logic for resolving address to function name

[Yordan Karadzhov]

On 15.04.22 г. 7:10 ч., Tzvetomir Stoyanov (VMware) wrote:
> Resolving virtual address to function name and vise versa is useful
> functionality for a trace application. Trace-cruncher can use it in two
> use cases:
>   - Resolving VMA to function name, when collecting user application
>     performance traces with perf.
>   - Resolving function name to VMA, when using ftarce uprobe dynamic
>     events.
> 
> Proposed implementation uses the bfd library to parse the binary files
> and read the symbol table. This information is available only if the
> files are not stripped.
> 
> Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
> ---
>   src/trace-obj-debug.c | 982 ++++++++++++++++++++++++++++++++++++++++++
>   src/trace-obj-debug.h |  54 +++
>   2 files changed, 1036 insertions(+)
>   create mode 100644 src/trace-obj-debug.c
>   create mode 100644 src/trace-obj-debug.h
> 
> diff --git a/src/trace-obj-debug.c b/src/trace-obj-debug.c
> new file mode 100644
> index 0000000..a6da22f
> --- /dev/null
> +++ b/src/trace-obj-debug.c
> @@ -0,0 +1,982 @@
> +// SPDX-License-Identifier: LGPL-2.1
> +/*
> + * Copyright (C) 2020, VMware, Tzvetomir Stoyanov <tz.stoyanov@gmail.com>
> + *
> + */

Add here

#ifndef _GNU_SOURCE
/** Use GNU C Library. */
#define _GNU_SOURCE
#endif // _GNU_SOURCE

> +#include <stdlib.h>
> +#include <stdio.h>
> +#include <string.h>
> +#include <limits.h>
> +#include <errno.h>
> +#include <bfd.h>
> +#include <unistd.h>
> +#include <dlfcn.h>
> +#include <fnmatch.h>
> +#include <ctype.h>
> +
> +#include "trace-obj-debug.h"
> +
> +//#define DEBUG_INTERNALS
> +
> +/* Got from demangle.h */
> +#define DMGL_AUTO (1 << 8)
> +
> +struct debug_bfd_handle {
> +	bfd *bfd;
> +	unsigned long long addr_offset;
> +};
> +
> +enum match_type {
> +	MATCH_EXACT	= 0,
> +	MATH_WILDCARD	= 1,
> +};
> +
> +struct debug_symbols {
> +	struct debug_symbols		*next;
> +	struct dbg_trace_symbols	symbol;
> +	enum match_type			match;
> +};
> +
> +struct debug_file {
> +	struct debug_file		*next;
> +	char				*file_name;
> +
> +	/* Start and end address, where this file is mapped into the memory of the process. */
> +	unsigned long long		vmem_start;
> +	unsigned long long		vmem_end;
> +
> +	/* bfd library context for this file. */
> +	struct debug_bfd_handle	*dbg;
> +
> +	/* Symbols to resolve, search in this file only. */
> +	int				sym_count;
> +	struct debug_symbols		*sym;
> +};
> +
> +struct dbg_trace_context {
> +	/* PID or name of the process. */
> +	int				pid;
> +	char				*fname;
> +
> +	/* List of all libraries and object files, mapped in the memory of the process. */
> +	struct dbg_trace_pid_maps	*fmaps;
> +
> +	/* Symbols to resolve, search in all files. */
> +	int				sym_count;
> +	struct debug_symbols		*sym;
> +
> +	/* List of all libraries and object files, opened from bfd library for processing. */
> +	struct debug_file		*files;
> +};
> +
> +#define RESOLVE_NAME		(1 << 0)
> +#define RESOLVE_VMA		(1 << 1)
> +#define RESOLVE_FOFFSET		(1 << 2)
> +
> +struct debug_obj_job {
> +	unsigned int flags;
> +	unsigned long long addr_offset;
> +	struct debug_symbols *symbols;
> +};
> +
> +struct debug_dwarf_bfd_context {
> +	asymbol **table;
> +	struct debug_obj_job *job;
> +};
> +
> +/*
> + * A hook, called from bfd library for each section of the file.
> + * The logic is used for symbol name and file offset resolving from given symbol VMA.
> + */
> +static void process_bfd_section(bfd *abfd, asection *section, void *param)
> +{
> +	struct debug_dwarf_bfd_context *context = (struct debug_dwarf_bfd_context *)param;
> +	unsigned int discriminator;
> +	const char *functionname;
> +	struct debug_symbols *s;
> +	unsigned long long vma;
> +	const char *filename;
> +	unsigned int line;
> +	bfd_boolean found;
> +
> +	/* Skip sections that have no code. */
> +	if (!(section->flags & SEC_CODE))
> +		return;
> +
> +	/* Loop through all symbols, that have to be resolved. */
> +	for (s = context->job->symbols; s; s = s->next) {
> +		if (s->symbol.vma_near)
> +			vma = s->symbol.vma_near;
> +		else if (s->symbol.vma_start)
> +			vma = s->symbol.vma_start;
> +		else
> +			continue;
> +
> +		/* Adjust the symbol's VMA, if this section is dynamically loaded. */
> +		if (abfd->flags & DYNAMIC)
> +			vma -=  context->job->addr_offset;
                                ^extra space.

Thanks!
Yordan

> +
> +		/* Check if requested symbol's vma is within the current section. */
> +		if (vma && section->vma <= vma &&
> +		    (section->vma + section->size) > vma) {
> +
> +			/* Found the file, where the symbol is defined. */
> +			if (!s->symbol.fname)
> +				s->symbol.fname = strdup(abfd->filename);
> +
> +			/* Found the offset of the symbol within the file. */
> +			if (context->job->flags & RESOLVE_FOFFSET)
> +				s->symbol.foffset = section->filepos + (vma - section->vma);
> +
> +			/* Look for the nearest function. */
> +			if (!s->symbol.name && (context->job->flags & RESOLVE_NAME)) {
> +				found = bfd_find_nearest_line_discriminator(abfd, section, context->table,
> +									    vma - section->vma, &filename,
> +									    &functionname, &line, &discriminator);
> +#ifdef DEBUG_INTERNALS
> +				printf("Find addr near 0x%X, offset 0x%X - > vma - 0x%X in %s, found %s\n\r",
> +					s->symbol.vma_near, context->job->addr_offset, vma, abfd->filename,
> +					found ? functionname : "NA");
> +#endif
> +				if (found) {
> +					/* Demangle the name of the function. */
> +					s->symbol.name = bfd_demangle(abfd, functionname, DMGL_AUTO);
> +					/* Found the name of the symbol. */
> +					if (!s->symbol.name)
> +						s->symbol.name = strdup(functionname);
> +				}
> +			}
> +		}
> +	}
> +}
> +
> +/* Load the symbol table from the file. */
> +static asymbol **get_sym_table(bfd *handle)
> +{
> +	long size, ssize, dsize;
> +	asymbol **symtable;
> +	long count;
> +
> +	if ((bfd_get_file_flags(handle) & HAS_SYMS) == 0)
> +		return NULL;
> +
> +	dsize = bfd_get_dynamic_symtab_upper_bound(handle);
> +	size = dsize > 0 ? dsize : 0;
> +
> +	ssize = bfd_get_symtab_upper_bound(handle);
> +	size += ssize > 0 ? ssize : 0;
> +
> +	if (size <= 0)
> +		return NULL;
> +
> +	symtable = (asymbol **) calloc(1, size);
> +	if (!symtable)
> +		return NULL;
> +
> +	count = bfd_canonicalize_symtab(handle, symtable);
> +	count += bfd_canonicalize_dynamic_symtab(handle, symtable + count);
> +	if (count <= 0) {
> +		free(symtable);
> +		return NULL;
> +	}
> +
> +	return symtable;
> +}
> +
> +/* Match the requested name to the name of the symbol. Handle a wildcard match. */
> +static bool symbol_match(char *pattern, enum match_type match, const char *symbol)
> +{
> +	bool ret = false;
> +
> +	switch (match) {
> +	case MATCH_EXACT:
> +		if (strlen(pattern) == strlen(symbol) &&
> +		    !strcmp(pattern, symbol))
> +			ret = true;
> +		break;
> +	case MATH_WILDCARD:
> +		if (!fnmatch(pattern, symbol, 0))
> +			ret = true;
> +		break;
> +	}
> +
> +	return ret;
> +}
> +
> +/* Lookup in the file's symbol table.
> + * The logic is used for symbol VMA resolving from given symbol name.
> + */
> +static int lookup_bfd_sym(struct debug_dwarf_bfd_context *context)
> +{
> +	struct debug_symbols *s, *last = NULL;
> +	struct debug_symbols *new, *new_list = NULL;
> +	unsigned long long vma;
> +	asymbol **sp;
> +	int res = 0;
> +
> +	for (sp = context->table; *sp != NULL; sp++) {
> +		/* Skip the symbol, if it is not a function. */
> +		if (!((*sp)->flags & BSF_FUNCTION))
> +			continue;
> +		/* Loop through all symbols that should be resolved. */
> +		for (s = context->job->symbols; s; s = s->next) {
> +			if (!s->symbol.name)
> +				continue;
> +			last = s;
> +			if (!symbol_match(s->symbol.name, s->match, (*sp)->name))
> +				continue;
> +#ifdef DEBUG_INTERNALS
> +			printf("Matched %s, pattern %s\n\r", (*sp)->name, s->symbol.name);
> +#endif
> +			vma = (*sp)->value + (*sp)->section->vma;
> +			/* Adjust the VMA, if the section is dynamically loaded. */
> +			if ((*sp)->the_bfd->flags & DYNAMIC)
> +				vma += context->job->addr_offset;
> +			if (s->match == MATCH_EXACT) {
> +				/* Exact match, update the VMA. */
> +				s->symbol.vma_start = vma;
> +			} else if (s->match == MATH_WILDCARD) {
> +				/* Wildcard pattern match, create a new symbol. */
> +				new = calloc(1, sizeof(struct debug_symbols));
> +				if (!new)
> +					break;
> +				new->symbol.name = strdup((*sp)->name);
> +				new->symbol.vma_start = vma;
> +				new->symbol.vma_near = s->symbol.vma_near;
> +				new->symbol.foffset = s->symbol.foffset;
> +				new->symbol.cookie = s->symbol.cookie;
> +				if (s->symbol.fname)
> +					new->symbol.fname = strdup(s->symbol.fname);
> +				new->next = new_list;
> +				new_list = new;
> +			}
> +			res++;
> +		}
> +	}
> +	if (last && !last->next)
> +		last->next = new_list;
> +
> +	return res;
> +}
> +
> +/* Process a bfd object from the file. */
> +static int process_bfd_object(bfd *abfd, struct debug_obj_job *job)
> +{
> +	struct debug_dwarf_bfd_context context;
> +	int ret = 0;
> +
> +	memset(&context, 0, sizeof(context));
> +	context.job = job;
> +	if (bfd_check_format_matches(abfd, bfd_object, NULL) ||
> +	    bfd_check_format_matches(abfd, bfd_core, NULL)) {
> +		context.table = get_sym_table(abfd);
> +		if (!context.table)
> +			return -1;
> +
> +		/* Resolve VMA from the symbol table. */
> +		if (job->flags & RESOLVE_VMA)
> +			lookup_bfd_sym(&context);
> +
> +		/* Resolve symbol name and file offset from file's sections. */
> +		if ((job->flags & RESOLVE_NAME) || (job->flags & RESOLVE_FOFFSET))
> +			bfd_map_over_sections(abfd, process_bfd_section, &context);
> +
> +		free(context.table);
> +	} else {
> +		ret = -1;
> +	}
> +
> +	return ret;
> +}
> +
> +/* Open a bfd archive file and read all objects. */
> +static int read_all_bfd(bfd *abfd, struct debug_obj_job *job)
> +{
> +	bfd *last_arfile = NULL;
> +	bfd *arfile = NULL;
> +	int ret = 0;
> +
> +	if (bfd_check_format(abfd, bfd_archive)) {
> +		for (;;) {
> +			bfd_set_error(bfd_error_no_error);
> +			arfile = bfd_openr_next_archived_file(abfd, arfile);
> +			if (!arfile) {
> +				if (bfd_get_error() != bfd_error_no_more_archived_files)
> +					break;
> +			}
> +			ret = read_all_bfd(arfile, job);
> +			if (last_arfile)
> +				bfd_close(last_arfile);
> +			last_arfile = arfile;
> +		}
> +		if (last_arfile)
> +			bfd_close(last_arfile);
> +	} else
> +		ret = process_bfd_object(abfd, job);
> +
> +	return ret;
> +}
> +
> +/**
> + * resolve_symbol_vma - name -> (vma, file offset) resolving
> + * @obj - pointer to object, returned by trace_obj_debug_create()
> + * @symbols - link list with desired symbols, with given name
> + *
> + * Get VMA and file offset of the symbols with given name.
> + * Return 0 on success, -1 on error.
> + */
> +static int resolve_symbol_vma(struct debug_bfd_handle *obj,
> +			      struct debug_symbols *symbols)
> +{
> +	struct debug_obj_job job;
> +	int ret;
> +
> +	memset(&job, 0, sizeof(job));
> +	job.flags |= RESOLVE_VMA;
> +	job.flags |= RESOLVE_FOFFSET;
> +	job.symbols = symbols;
> +	job.addr_offset = obj->addr_offset;
> +	ret = read_all_bfd(obj->bfd, &job);
> +
> +	return ret;
> +}
> +
> +/**
> + * resolve_symbol_name - vma -> name resolving
> + * @obj - pointer to object, returned by trace_obj_debug_create()
> + * @symbols - link list with desired symbols, with given VMA
> + *
> + * Get names of the symbols with given VMA, look for nearest symbol to that VMA.
> + * Return 0 on success, -1 on error.
> + */
> +static int resolve_symbol_name(struct debug_bfd_handle *obj,
> +			       struct debug_symbols *symbols)
> +{
> +	struct debug_obj_job job;
> +
> +	if (!obj || !obj->bfd)
> +		return -1;
> +	memset(&job, 0, sizeof(job));
> +	job.flags |= RESOLVE_NAME;
> +	job.addr_offset = obj->addr_offset;
> +	job.symbols = symbols;
> +	return read_all_bfd(obj->bfd, &job);
> +}
> +
> +/**
> + * debug_handle_destroy - Close file opened with trace_obj_debug_create()
> + * @obj - pointer to object, returned by trace_obj_debug_create()
> + *
> + * Close the file and free any allocated resources, related to file's debug
> + * information.
> + */
> +static void debug_handle_destroy(struct debug_bfd_handle *obj)
> +{
> +	if (obj && obj->bfd)
> +		bfd_close(obj->bfd);
> +	free(obj);
> +}
> +
> +/**
> + * debug_handle_create - Open binary file for parsing ELF and DWARF information
> + * @name: Name of the binary ELF file.
> + *
> + * Return pointer to trace_obj_debug structure, that can be passed to other APIs
> + * for extracting debug information from the file. NULL in case of an error.
> + */
> +static struct debug_bfd_handle *debug_handle_create(char *file)
> +{
> +	struct debug_bfd_handle *obj = NULL;
> +
> +	obj = calloc(1, sizeof(*obj));
> +	if (!obj)
> +		return NULL;
> +
> +	bfd_init();
> +	obj->bfd = bfd_openr(file, NULL);
> +	if (!obj->bfd)
> +		goto error;
> +	obj->bfd->flags |= BFD_DECOMPRESS;
> +
> +	return obj;
> +
> +error:
> +	debug_handle_destroy(obj);
> +	return NULL;
> +}
> +
> +/* Get the full path of process's executable, using the /proc fs. */
> +static char *get_full_name(int pid)
> +{
> +	char mapname[PATH_MAX+1];
> +	char fname[PATH_MAX+1];
> +	int ret;
> +
> +	sprintf(fname, "/proc/%d/exe", pid);
> +	ret = readlink(fname, mapname, PATH_MAX);
> +	if (ret >= PATH_MAX || ret < 0)
> +		return NULL;
> +	mapname[ret] = 0;
> +
> +	return strdup(mapname);
> +}
> +
> +/* Get or create a bfd debug context for an object file. */
> +static struct debug_file *get_mapped_file(struct dbg_trace_context *dbg,
> +					  char *fname,
> +					  unsigned long long vmem_start)
> +{
> +	struct debug_file *file = dbg->files;
> +
> +	/* Search if the file is already added. */
> +	while (file) {
> +		if (!strcmp(fname, file->file_name) &&
> +		    vmem_start && file->vmem_end == vmem_start)
> +			return file;
> +		file = file->next;
> +	}
> +
> +	file = calloc(1, sizeof(*file));
> +	if (!file)
> +		return NULL;
> +	file->file_name = strdup(fname);
> +	if (!file->file_name)
> +		goto error;
> +	file->dbg = debug_handle_create(fname);
> +	file->next = dbg->files;
> +	dbg->files = file;
> +	return file;
> +
> +error:
> +	free(file->file_name);
> +	debug_handle_destroy(file->dbg);
> +	free(file);
> +	return NULL;
> +}
> +
> +/* Destroy a bfd debug context. */
> +void dbg_trace_context_destroy(struct dbg_trace_context *dbg)
> +{
> +	struct debug_file *fdel;
> +	struct debug_symbols *sdel;
> +
> +	while (dbg->sym) {
> +		sdel = dbg->sym;
> +		dbg->sym = dbg->sym->next;
> +		free(sdel->symbol.name);
> +		free(sdel->symbol.fname);
> +		free(sdel);
> +	}
> +	while (dbg->files) {
> +		fdel = dbg->files;
> +		dbg->files = dbg->files->next;
> +		debug_handle_destroy(fdel->dbg);
> +		while (fdel->sym) {
> +			sdel = fdel->sym;
> +			fdel->sym = fdel->sym->next;
> +			free(sdel->symbol.name);
> +			free(sdel->symbol.fname);
> +			free(sdel);
> +		}
> +		free(fdel);
> +	}
> +
> +	free(dbg->fname);
> +	dbg_trace_free_filemap(dbg->fmaps);
> +	free(dbg);
> +}
> +
> +/* Add an object file, mapped to specific memory of the process. */
> +int dbg_trace_context_add_file(struct dbg_trace_context *dbg, char *file_name,
> +			       unsigned long long vmem_start,
> +			       unsigned long long vmem_end,
> +			       unsigned long long pgoff)
> +{
> +	struct debug_file *file;
> +
> +	file = get_mapped_file(dbg, file_name, vmem_start);
> +	if (!file)
> +		return -1;
> +	if (file->vmem_end == vmem_start) {
> +		file->vmem_end = vmem_end;
> +	} else {
> +		file->vmem_start = vmem_start;
> +		file->vmem_end = vmem_end;
> +		if (file->dbg)
> +			file->dbg->addr_offset = vmem_start - pgoff;
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * dbg_trace_context_create_pid - create debug context for given PID
> + * @pid - ID of running process
> + * @libs - if true: inspect also all libraries, uased by the given process.
> + *
> + * Returns a pointer to allocated debug context, or NULL in case of an error.
> + */
> +struct dbg_trace_context *dbg_trace_context_create_pid(int pid, bool libs)
> +{
> +	struct dbg_trace_context *dbg;
> +	unsigned int i;
> +
> +	dbg = calloc(1, sizeof(*dbg));
> +	if (!dbg)
> +		return NULL;
> +
> +	dbg->pid = pid;
> +	/* Get the full path of the process executable. */
> +	dbg->fname = get_full_name(pid);
> +	if (!dbg->fname) {
> +		free(dbg);
> +		return NULL;
> +	}
> +
> +	/* Get the memory map of all libraries, linked to the process. */
> +	dbg_trace_get_filemap(&dbg->fmaps, pid);
> +
> +	for (i = 0; i < dbg->fmaps->nr_lib_maps; i++) {
> +		if (!libs && strcmp(dbg->fname, dbg->fmaps->lib_maps[i].lib_name))
> +			continue;
> +		/* Create a bfd debug object for each file. */
> +		dbg_trace_context_add_file(dbg, dbg->fmaps->lib_maps[i].lib_name,
> +					   dbg->fmaps->lib_maps[i].start,
> +					   dbg->fmaps->lib_maps[i].end, 0);
> +	}
> +
> +	return dbg;
> +}
> +
> +/* Get the full path of a library. */
> +static char *get_lib_full_path(char *libname)
> +{
> +	void *h = dlmopen(LM_ID_NEWLM, libname, RTLD_LAZY);
> +	char dldir[PATH_MAX+1];
> +	char *fname = NULL;
> +	int ret;
> +
> +	if (!h)
> +		return NULL;
> +
> +	ret = dlinfo(h, RTLD_DI_ORIGIN, dldir);
> +	dlclose(h);
> +
> +	if (ret || asprintf(&fname, "%s/%s", dldir, libname) <= 0)
> +		return NULL;
> +
> +	return fname;
> +}
> +
> +/* Get the memory map of all libraries, linked to an executable file. */
> +static int debug_obj_file_add_libs(struct dbg_trace_context *dbg,
> +				   struct debug_file *file)
> +{
> +	char line[PATH_MAX];
> +	char *libname;
> +	char *trimmed;
> +	char *fullname;
> +	FILE *fp = NULL;
> +	int ret = -1;
> +
> +	setenv("LD_TRACE_LOADED_OBJECTS", "1", 1);
> +	fp = popen(file->file_name, "r");
> +	if (!fp)
> +		goto out;
> +
> +	while (fgets(line, sizeof(line), fp) != NULL) {
> +		libname = strchr(line, ' ');
> +		trimmed = line;
> +		if (libname) {
> +			*libname = '\0';
> +			while (isspace(*trimmed))
> +				trimmed++;
> +			if (*trimmed != '/') {
> +				fullname = get_lib_full_path(trimmed);
> +				if (fullname) {
> +					get_mapped_file(dbg, fullname, 0);
> +					free(fullname);
> +				}
> +			} else {
> +				get_mapped_file(dbg, trimmed, 0);
> +			}
> +		}
> +	}
> +
> +out:
> +	unsetenv("LD_TRACE_LOADED_OBJECTS");
> +	if (fp)
> +		pclose(fp);
> +	return ret;
> +}
> +
> +/**
> + * dbg_trace_context_create_file - create debug context for given executable file
> + * @fname - full path to an executable file
> + * @libs - if true: inspect also all libraries, used by the given file.
> + *
> + * Returns a pointer to allocated debug context, or NULL in case of an error.
> + */
> +struct dbg_trace_context *dbg_trace_context_create_file(char *fname, bool libs)
> +{
> +	struct dbg_trace_context *dbg;
> +	struct debug_file *file;
> +
> +	dbg = calloc(1, sizeof(*dbg));
> +	if (!dbg)
> +		return NULL;
> +
> +	dbg->fname = strdup(fname);
> +	file = get_mapped_file(dbg, fname, 0);
> +	if (!file)
> +		goto error;
> +	if (libs)
> +		debug_obj_file_add_libs(dbg, file);
> +
> +#ifdef DEBUG_INTERNALS
> +	printf("Created debug object for %s:\n\r", dbg->fname);
> +	file = dbg->files;
> +	while (file) {
> +		printf("\t%s\n\r", file->file_name);
> +		file = file->next;
> +	}
> +#endif
> +	return dbg;
> +
> +error:
> +	dbg_trace_context_destroy(dbg);
> +	return NULL;
> +}
> +
> +static void set_unknown(struct debug_symbols *sym, char *file)
> +{
> +	while (sym) {
> +		if (!sym->symbol.fname)
> +			sym->symbol.fname = strdup(file);
> +		sym = sym->next;
> +	}
> +}
> +
> +/* Perform the requested symbols resolving, using the bfd library. */
> +int dbg_trace_resolve_symbols(struct dbg_trace_context *obj)
> +{
> +	struct debug_file *file;
> +
> +	for (file = obj->files; file; file = file->next) {
> +		if (!file->dbg) {
> +			set_unknown(file->sym, file->file_name);
> +			continue;
> +		}
> +		/* Resolve near VMA -> name. */
> +		resolve_symbol_name(file->dbg, file->sym);
> +		/* Resolve name -> exact VMA. */
> +		resolve_symbol_vma(file->dbg, file->sym);
> +		resolve_symbol_vma(file->dbg, obj->sym);
> +	}
> +
> +	return 0;
> +}
> +
> +/* Add VMA -> name resolving request. */
> +static int add_resolve_vma2name(struct dbg_trace_context *obj,
> +				unsigned long long vma, int cookie)
> +{
> +	struct debug_symbols *s = NULL;
> +	struct debug_file *file;
> +
> +	file = obj->files;
> +	while (file) {
> +		/* Find the file, where the requested VMA is. */
> +		if (vma >= file->vmem_start && vma <= file->vmem_end)
> +			break;
> +		file = file->next;
> +	}
> +
> +	if (file) {
> +		s = file->sym;
> +		while (s) {
> +			/* Check if the given VMA is already added for resolving. */
> +			if (s->symbol.vma_near == vma)
> +				break;
> +			s = s->next;
> +		}
> +		if (!s) {
> +			s = calloc(1, sizeof(*s));
> +			if (!s)
> +				return -1;
> +			s->symbol.cookie = cookie;
> +			s->symbol.vma_near = vma;
> +			s->symbol.fname = strdup(file->file_name);
> +			if (!s->symbol.fname)
> +				goto error;
> +			s->next = file->sym;
> +			file->sym = s;
> +			file->sym_count++;
> +		}
> +	}
> +
> +	if (s)
> +		return 0;
> +error:
> +	if (s) {
> +		free(s->symbol.fname);
> +		free(s);
> +	}
> +	return -1;
> +}
> +
> +/* Add name - VMA resolving request, The @name can have wildcards. */
> +static int add_resolve_name2vma(struct dbg_trace_context *obj, char *name, int cookie)
> +{
> +	struct debug_symbols *s = NULL;
> +
> +	s = obj->sym;
> +	while (s) {
> +		/* Check if the given name is already added for resolving. */
> +		if (s->symbol.name && !strcmp(name, s->symbol.name))
> +			break;
> +		s = s->next;
> +	}
> +	if (!s) {
> +		s = calloc(1, sizeof(*s));
> +		if (!s)
> +			return -1;
> +		s->symbol.cookie = cookie;
> +		s->symbol.name = strdup(name);
> +		if (!s->symbol.name)
> +			goto error;
> +		if (strchr(name, '*') || strchr(name, '?'))
> +			s->match = MATH_WILDCARD;
> +
> +		s->next = obj->sym;
> +		obj->sym = s;
> +		obj->sym_count++;
> +	}
> +
> +	return 0;
> +
> +error:
> +	if (s) {
> +		free(s->symbol.name);
> +		free(s);
> +	}
> +	return -1;
> +}
> +
> +/**
> + * dbg_trace_add_resolve_symbol - add new resolving request
> + * @obj - debug object context
> + * @vma - VMA->name resolving, if @vma is not 0
> + * @name - name-VMA resolving, if @name is not NULL
> + * @cookie - a cookie, attached to each successful resolving from this request
> + *
> + * Returns 0 if the request is added successfully, or -1 in case of an error.
> + */
> +int dbg_trace_add_resolve_symbol(struct dbg_trace_context *obj,
> +				 unsigned long long vma, char *name, int cookie)
> +{
> +	int ret = -1;
> +
> +	if (!obj)
> +		return -1;
> +
> +	if (!name && vma) /* vma -> name resolving */
> +		ret = add_resolve_vma2name(obj, vma, cookie);
> +	else if (name) /* name -> vma resolving */
> +		ret = add_resolve_name2vma(obj, name, cookie);
> +
> +	return ret;
> +}
> +
> +static int walk_symbols(struct debug_symbols *sym,
> +			int (*callback)(struct dbg_trace_symbols *, void *),
> +			void *context)
> +{
> +	while (sym) {
> +		if (callback(&sym->symbol, context))
> +			return -1;
> +		sym = sym->next;
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * dbg_trace_walk_resolved_symbols - walk through all resolved symbols
> + * @obj - debug object context
> + * @callback - a callback hook, called for each resolved symbol.
> + *		If the callback returns non-zero, the walk stops.
> + * @context - a user specified context, passed to the callback
> + */
> +void dbg_trace_walk_resolved_symbols(struct dbg_trace_context *obj,
> +				     int (*callback)(struct dbg_trace_symbols *, void *),
> +				     void *context)
> +{
> +	struct debug_file *file;
> +
> +	walk_symbols(obj->sym, callback, context);
> +	file = obj->files;
> +	while (file) {
> +		walk_symbols(file->sym, callback, context);
> +		file = file->next;
> +	}
> +}
> +
> +/**
> + * trace_debug_free_symbols - free array of debug symbols
> + * @symbols - array with debug symbols
> + * @count - count of the @symbols array
> + */
> +void trace_debug_free_symbols(struct dbg_trace_symbols *symbols, int count)
> +{
> +	int i;
> +
> +	if (!symbols)
> +		return;
> +
> +	for (i = 0; i < count; i++) {
> +		free(symbols[i].name);
> +		free(symbols[i].fname);
> +	}
> +	free(symbols);
> +
> +}
> +
> +#define _STRINGIFY(x) #x
> +#define STRINGIFY(x) _STRINGIFY(x)
> +/**
> + * dbg_trace_get_filemap - get a memory map of a process, using /proc fs
> + * @pid_maps - return: list of files, mapped into the process memory
> + * @pid - id of a process
> + *
> + * Returns 0 on success, -1 in case of an error.
> + */
> +int dbg_trace_get_filemap(struct dbg_trace_pid_maps **pid_maps, int pid)
> +{
> +	struct dbg_trace_pid_maps *maps = *pid_maps;
> +	struct dbg_trace_proc_addr_map *map;
> +	unsigned long long begin, end;
> +	struct dbg_trace_pid_maps *m;
> +	char mapname[PATH_MAX+1];
> +	char fname[PATH_MAX+1];
> +	char buf[PATH_MAX+100];
> +	unsigned int i;
> +	FILE *f;
> +	int ret;
> +	int res;
> +
> +	sprintf(fname, "/proc/%d/exe", pid);
> +	ret = readlink(fname, mapname, PATH_MAX);
> +	if (ret >= PATH_MAX || ret < 0)
> +		return -ENOENT;
> +	mapname[ret] = 0;
> +
> +	sprintf(fname, "/proc/%d/maps", pid);
> +	f = fopen(fname, "r");
> +	if (!f)
> +		return -ENOENT;
> +
> +	while (maps) {
> +		if (pid == maps->pid)
> +			break;
> +		maps = maps->next;
> +	}
> +
> +	ret = -ENOMEM;
> +	if (!maps) {
> +		maps = calloc(1, sizeof(*maps));
> +		if (!maps)
> +			goto out_fail;
> +		maps->pid = pid;
> +		maps->next = *pid_maps;
> +		*pid_maps = maps;
> +	} else {
> +		for (i = 0; i < maps->nr_lib_maps; i++)
> +			free(maps->lib_maps[i].lib_name);
> +		free(maps->lib_maps);
> +		maps->lib_maps = NULL;
> +		maps->nr_lib_maps = 0;
> +		free(maps->proc_name);
> +	}
> +
> +	maps->proc_name = strdup(mapname);
> +	if (!maps->proc_name)
> +		goto out_fail;
> +
> +	while (fgets(buf, sizeof(buf), f)) {
> +		mapname[0] = '\0';
> +		res = sscanf(buf, "%llx-%llx %*s %*x %*s %*d %"STRINGIFY(PATH_MAX)"s",
> +			     &begin, &end, mapname);
> +		if (res == 3 && mapname[0] != '\0') {
> +			map = realloc(maps->lib_maps,
> +				      (maps->nr_lib_maps + 1) * sizeof(*map));
> +			if (!map)
> +				goto out_fail;
> +			map[maps->nr_lib_maps].end = end;
> +			map[maps->nr_lib_maps].start = begin;
> +			map[maps->nr_lib_maps].lib_name = strdup(mapname);
> +			if (!map[maps->nr_lib_maps].lib_name)
> +				goto out_fail;
> +			maps->lib_maps = map;
> +			maps->nr_lib_maps++;
> +		}
> +	}
> +
> +	fclose(f);
> +	return 0;
> +
> +out_fail:
> +	fclose(f);
> +	if (maps) {
> +		for (i = 0; i < maps->nr_lib_maps; i++)
> +			free(maps->lib_maps[i].lib_name);
> +		if (*pid_maps != maps) {
> +			m = *pid_maps;
> +			while (m) {
> +				if (m->next == maps) {
> +					m->next = maps->next;
> +					break;
> +				}
> +				m = m->next;
> +			}
> +		} else
> +			*pid_maps = maps->next;
> +		free(maps->lib_maps);
> +		maps->lib_maps = NULL;
> +		maps->nr_lib_maps = 0;
> +		free(maps->proc_name);
> +		maps->proc_name = NULL;
> +		free(maps);
> +	}
> +	return ret;
> +}
> +
> +static void procmap_free(struct dbg_trace_pid_maps *maps)
> +{
> +	unsigned int i;
> +
> +	if (!maps)
> +		return;
> +	if (maps->lib_maps) {
> +		for (i = 0; i < maps->nr_lib_maps; i++)
> +			free(maps->lib_maps[i].lib_name);
> +		free(maps->lib_maps);
> +	}
> +	free(maps->proc_name);
> +	free(maps);
> +}
> +
> +/**
> + * dbg_trace_free_filemap - Free list of files, associated with given process
> + * @maps - list of files, returned by dbg_trace_get_filemap()
> + */
> +void dbg_trace_free_filemap(struct dbg_trace_pid_maps *maps)
> +{
> +	struct dbg_trace_pid_maps *del;
> +
> +	while (maps) {
> +		del = maps;
> +		maps = maps->next;
> +		procmap_free(del);
> +	}
> +}
> diff --git a/src/trace-obj-debug.h b/src/trace-obj-debug.h
> new file mode 100644
> index 0000000..19091ad
> --- /dev/null
> +++ b/src/trace-obj-debug.h
> @@ -0,0 +1,54 @@
> +/* SPDX-License-Identifier: LGPL-2.1 */
> +
> +/*
> + * Copyright 2022 VMware Inc, Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
> + */
> +
> +#ifndef _TC_TRACE_DEBUG_UTILS_
> +#define _TC_TRACE_DEBUG_UTILS_
> +
> +/* --- Debug symbols--- */
> +struct dbg_trace_pid_maps {
> +	struct dbg_trace_pid_maps	*next;
> +	struct dbg_trace_proc_addr_map	*lib_maps;
> +	unsigned int			nr_lib_maps;
> +	char				*proc_name;
> +	int				pid;
> +};
> +int dbg_trace_get_filemap(struct dbg_trace_pid_maps **file_maps, int pid);
> +void dbg_trace_free_filemap(struct dbg_trace_pid_maps *maps);
> +
> +struct dbg_trace_symbols {
> +	char *name;			/* symbol's name */
> +	char *fname;			/* symbol's file */
> +	int cookie;
> +	unsigned long long vma_start;	/* symbol's start VMA */
> +	unsigned long long vma_near;	/* symbol's requested VMA */
> +	unsigned long long foffset;	/* symbol's offset in the binary file*/
> +};
> +
> +struct dbg_trace_proc_addr_map {
> +	unsigned long long	start;
> +	unsigned long long	end;
> +	char			*lib_name;
> +};
> +
> +struct dbg_trace_context;
> +struct dbg_trace_context *dbg_trace_context_create_file(char *file, bool libs);
> +struct dbg_trace_context *dbg_trace_context_create_pid(int pid, bool libs);
> +
> +void dbg_trace_context_destroy(struct dbg_trace_context *debug);
> +int dbg_trace_context_add_file(struct dbg_trace_context *dbg, char *file_name,
> +			       unsigned long long vmem_start,
> +			       unsigned long long vmem_end,
> +			       unsigned long long pgoff);
> +
> +int dbg_trace_resolve_symbols(struct dbg_trace_context *obj);
> +int dbg_trace_add_resolve_symbol(struct dbg_trace_context *obj,
> +				 unsigned long long vma, char *name, int cookie);
> +
> +void dbg_trace_walk_resolved_symbols(struct dbg_trace_context *obj,
> +				     int (*callback)(struct dbg_trace_symbols *, void *),
> +				     void *context);
> +
> +#endif /* _TC_TRACE_DEBUG_UTILS_ */

Re: [RFC PATCH v2 3/4] trace-cruncher: High level wrappers for ftrace uprobes

[Yordan Karadzhov]

On 15.04.22 г. 7:10 ч., Tzvetomir Stoyanov (VMware) wrote:
> Using uprobes requires finding the offset of a user function within the
> binary file, where this functions is compiled. This is not a trivial
> task, especially in the cases when a bunch of uprobes to user functions
> should be added.
> A high level trace-cruncher API allows adding multiple user functions as
> uprobes or uretprobes. It supports wildcards for function names and
> adding uprobes for library functions, used by the applications.
> 
> Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
> ---
>   setup.py             |   4 +-
>   src/ftracepy-utils.c | 628 +++++++++++++++++++++++++++++++++++++++++++
>   src/ftracepy-utils.h |  17 ++
>   src/ftracepy.c       |  35 +++
>   4 files changed, 682 insertions(+), 2 deletions(-)
> 
> diff --git a/setup.py b/setup.py
> index 58561cf..464d3d2 100644
> --- a/setup.py
> +++ b/setup.py
> @@ -71,8 +71,8 @@ def extension(name, sources, libraries):
>   
>   def main():
>       module_ft = extension(name='tracecruncher.ftracepy',
> -                          sources=['src/ftracepy.c', 'src/ftracepy-utils.c'],
> -                          libraries=['traceevent', 'tracefs'])
> +                          sources=['src/ftracepy.c', 'src/ftracepy-utils.c', 'src/trace-obj-debug.c'],
> +                          libraries=['traceevent', 'tracefs', 'bfd'])
>   
>       cythonize('src/npdatawrapper.pyx', language_level = 3)
>       module_data = extension(name='tracecruncher.npdatawrapper',
> diff --git a/src/ftracepy-utils.c b/src/ftracepy-utils.c
> index b39459b..d420dce 100644
> --- a/src/ftracepy-utils.c
> +++ b/src/ftracepy-utils.c
> @@ -15,15 +15,46 @@
>   #include <sys/utsname.h>
>   #include <sys/wait.h>
>   #include <signal.h>
> +#include <semaphore.h>
>   #include <time.h>
>   
>   // trace-cruncher
>   #include "ftracepy-utils.h"
> +#include "trace-obj-debug.h"
>   
>   PyObject *TFS_ERROR;
>   PyObject *TEP_ERROR;
>   PyObject *TRACECRUNCHER_ERROR;
>   
> +#define UPROBES_SYSTEM "tc_uprobes"

Note that we already have

#define TC_SYS	"tcrunch"

Does it make sence to define a second trace-cruncher system?

> +
> +#define FTRACE_UPROBE		0x1
> +#define FTRACE_URETPROBE	0x2
> +
> +struct fprobes_list {
> +	int size;
> +	int count;
> +	void **data;
> +};
> +
> +struct utrace_func {
> +	int type;
> +	char *func_name;
> +	char *func_args;
> +};
> +
> +struct py_utrace_context {
> +	pid_t pid;
> +	char *fname;
> +	char **argv;
> +	char *usystem;
> +	uint32_t trace_time; /* in msec */
> +	struct fprobes_list fretprobes;
> +	struct fprobes_list ufuncs;
> +	struct fprobes_list uevents;
> +	struct dbg_trace_context *dbg;
> +};
> +
>   static char *kernel_version()
>   {
>   	struct utsname uts;
> @@ -3508,3 +3539,600 @@ void PyFtrace_at_exit(void)
>   	if (seq.buffer)
>   		trace_seq_destroy(&seq);
>   }
> +
> +#define INIT_SIZE	100
This name is too general. Try to make it clear what is to be initialized with this size.

> +static int utrace_list_add(struct fprobes_list *list, void *data)
> +{
> +	void **tmp;
> +	int size;
> +
> +	if (list->size <= list->count) {
> +		if (!list->size)
> +			size = INIT_SIZE;
> +		else
> +			size = 2 * list->size;
> +		tmp = realloc(list->data, size * sizeof(void *));
> +		if (!tmp)
> +			return -1;
> +		list->data = tmp;
> +		list->size = size;
> +	}
> +
> +	list->data[list->count] = data;
> +	list->count++;
> +	return list->count - 1;
> +}
> +
> +void py_utrace_free(struct py_utrace_context *utrace)
> +{
> +	struct utrace_func *f;
> +	int i;
> +
> +	if (!utrace)
> +		return;
> +	if (utrace->dbg)
> +		dbg_trace_context_destroy(utrace->dbg);
> +
> +	for (i = 0; i < utrace->ufuncs.count; i++) {
> +		f = utrace->ufuncs.data[i];
> +		free(f->func_name);
> +		free(f);
> +	}
> +	free(utrace->ufuncs.data);
> +	if (utrace->argv) {
> +		i = 0;
> +		while (utrace->argv[i])
> +			free(utrace->argv[i++]);
> +		free(utrace->argv);
> +	}
> +
> +	for (i = 0; i < utrace->uevents.count; i++)
> +		tracefs_dynevent_free(utrace->uevents.data[i]);
> +	free(utrace->uevents.data);
> +
> +	free(utrace->fname);
> +	free(utrace->usystem);
> +	free(utrace);
> +}
> +
> +/*
> + * All strings, used as ftrace system or event name must contain only
> + * alphabetic characters, digits or underscores.
> + */
> +static void fname_unify(char *fname)
> +{
> +	int i;
> +
> +	for (i = 0; fname[i]; i++)
> +		if (!isalnum(fname[i]) && fname[i] != '_')
> +			fname[i] = '_';
> +}
> +
> +int py_utrace_destroy(struct py_utrace_context *utrace)
> +{
> +	int i;
> +
> +	for (i = 0; i < utrace->uevents.count; i++)
> +		tracefs_dynevent_destroy(utrace->uevents.data[i], true);
> +
> +	return 0;
> +}
> +
> +static struct py_utrace_context *utrace_new(pid_t pid, char *fname, char **argv, bool libs)
> +{
> +	struct py_utrace_context *utrace;
> +	char *file;
> +
> +	utrace = calloc(1, sizeof(*utrace));
> +	if (!utrace)
> +		return NULL;
> +
> +	if (fname) {
> +		utrace->argv = argv;
> +		utrace->dbg = dbg_trace_context_create_file(fname, libs);
> +		if (!utrace->dbg)
> +			goto error;
> +		utrace->fname = strdup(fname);
> +		if (!utrace->fname)
> +			goto error;
> +		file = strrchr(fname, '/');
> +		if (file)
> +			file++;
> +		if (!file || *file == '\0')
> +			file = fname;
> +		if (asprintf(&utrace->usystem, "%s_%s", UPROBES_SYSTEM, file) <= 0)
> +			goto error;
> +	} else {
> +		utrace->pid = pid;
> +		utrace->dbg = dbg_trace_context_create_pid(pid, libs);
> +		if (!utrace->dbg)
> +			goto error;
> +		if (asprintf(&utrace->usystem, "%s_%d", UPROBES_SYSTEM, pid) <= 0)
> +			goto error;
> +	}
> +
> +	fname_unify(utrace->usystem);
> +	return utrace;
> +
> +error:
> +	py_utrace_free(utrace);
> +	return NULL;
> +}
> +
> +static int py_utrace_add_func(struct py_utrace_context *utrace, char *func, int type)
> +{
> +	struct utrace_func *p;
> +	int ret;
> +	int i;
> +
> +	for (i = 0; i < utrace->ufuncs.count; i++) {
> +		p = utrace->ufuncs.data[i];
> +		if (!strcmp(p->func_name, func)) {
> +			p->type |= type;
> +			return 0;
> +		}
> +	}
> +
> +	p = calloc(1, sizeof(*p));
> +	if (!p)
> +		return -1;
> +	p->func_name = strdup(func);
> +	if (!p->func_name)
> +		goto error;
> +	p->type = type;
> +
> +	ret = utrace_list_add(&utrace->ufuncs, p);
> +	if (ret < 0)
> +		goto error;
> +
> +	if (dbg_trace_add_resolve_symbol(utrace->dbg, 0, func, ret))
> +		goto error;
> +
> +	return 0;
> +
> +error:
> +	free(p->func_name);
> +	free(p);
> +	return -1;
> +}
> +
> +PyObject *PyUserTrace_add_function(PyUserTrace *self, PyObject *args,
> +				   PyObject *kwargs)
> +{
> +	struct py_utrace_context *utrace = self->ptrObj;
> +	static char *kwlist[] = {"fname", NULL};
> +	char *fname;
> +
> +	if (!PyArg_ParseTupleAndKeywords(args,
> +					 kwargs,
> +					 "s",
> +					 kwlist,
> +					 &fname)) {
> +		return NULL;
> +	}
> +
> +	if (py_utrace_add_func(utrace, fname, FTRACE_UPROBE) < 0) {
> +		MEM_ERROR
> +		return NULL;
> +	}
> +
> +	Py_RETURN_NONE;
> +}
> +
> +PyObject *PyUserTrace_add_ret_function(PyUserTrace *self, PyObject *args,
> +				       PyObject *kwargs)
> +{
> +	struct py_utrace_context *utrace = self->ptrObj;
> +	static char *kwlist[] = {"fname", NULL};
> +	char *fname;
> +
> +	if (!PyArg_ParseTupleAndKeywords(args,
> +					 kwargs,
> +					 "s",
> +					 kwlist,
> +					 &fname)) {
> +		return NULL;
> +	}
> +
> +	if (py_utrace_add_func(utrace, fname, FTRACE_URETPROBE) < 0) {
> +		MEM_ERROR
> +		return NULL;
> +	}
> +
> +	Py_RETURN_NONE;
> +}
> +
> +/*
> + * max event name is 64 bytes, hard coded in the kernel.
> + * it can consists only of alphabetic characters, digits or underscores
> + */
Please start comments with capital and end with period.


> +#define FILENAME_TRUNCATE	10
> +#define FUNCAME_TRUNCATE	50
> +static char *uprobe_event_name(char *file, char *func, int type)
> +{
> +	char *event = NULL;
> +	char *fname;
> +
> +	fname = strrchr(file, '/');
> +	if (fname)
> +		fname++;
> +	if (!fname || *fname == '\0')
> +		fname = file;
> +
> +	asprintf(&event, "%s%.*s_%.*s",
> +		 type == FTRACE_URETPROBE ? "r_":"",
> +		 FILENAME_TRUNCATE, fname, FUNCAME_TRUNCATE, func);
> +	if (event)
> +		fname_unify(event);
> +
> +	return event;
> +}
> +
> +/*
> + * Create uprobe based on function name,
> + * file name and function offset within the file
> + */
> +static int utrace_event_create(struct py_utrace_context *utrace,
> +			       struct dbg_trace_symbols *sym, char *fecthargs,
> +			       int type)
> +{
> +	struct tracefs_dynevent *uevent = NULL;
> +	char *rname;
> +
> +	/* Generate uprobe event name, according to ftrace name requirements */
> +	rname = uprobe_event_name(sym->fname, sym->name, type);
> +	if (!rname)
> +		return -1;
> +
> +	if (type == FTRACE_URETPROBE)
> +		uevent = tracefs_uretprobe_alloc(utrace->usystem, rname,
> +						 sym->fname, sym->foffset, fecthargs);
> +	else
> +		uevent = tracefs_uprobe_alloc(utrace->usystem, rname,
> +					      sym->fname, sym->foffset, fecthargs);
> +
> +	free(rname);
> +	if (!uevent)
> +		return -1;
> +
> +	if (tracefs_dynevent_create(uevent)) {
> +		tracefs_dynevent_free(uevent);
> +		return -1;
> +	}
> +
> +	utrace_list_add(&utrace->uevents, uevent);
> +	return 0;
> +}
> +
> +/* callback, called on each resolved function */
> +static int symblos_walk(struct dbg_trace_symbols *sym, void *context)
> +{
> +	struct py_utrace_context *utrace = context;
> +	struct utrace_func *ufunc;
> +
> +	if (!sym->name || !sym->fname || !sym->foffset ||
> +	    sym->cookie < 0 || sym->cookie >= utrace->ufuncs.count)
> +		return 0;
> +
> +	ufunc = utrace->ufuncs.data[sym->cookie];
> +
> +	if (ufunc->type & FTRACE_UPROBE)
> +		utrace_event_create(utrace, sym, ufunc->func_args, FTRACE_UPROBE);
> +
> +	if (ufunc->type & FTRACE_URETPROBE)
> +		utrace_event_create(utrace, sym, ufunc->func_args, FTRACE_URETPROBE);
> +
> +	return 0;
> +}
> +
> +static void py_utrace_generate_uprobes(struct py_utrace_context *utrace)
> +{
> +	/* Find the exact name and file offset of each user function that should be traced */
> +	dbg_trace_resolve_symbols(utrace->dbg);
> +	dbg_trace_walk_resolved_symbols(utrace->dbg, symblos_walk, utrace);
> +}
> +
> +static int start_trace(struct py_utrace_context *utrace, struct tracefs_instance *instance)

The name is too general.

> +{
> +	PyObject *pPid = PyLong_FromLong(utrace->pid);
> +	bool ret;
> +
> +	if (!pPid)
> +		return -1;
> +
> +	/* Filter the trace only on desired pid(s) */
> +	ret = hook2pid(instance, PyLong_FromLong(utrace->pid), true);
> +	Py_DECREF(pPid);
> +	if (!ret) {
> +		PyErr_SetString(TRACECRUNCHER_ERROR,
> +				"Failed to set trace filter");
> +		return -1;
> +	}
> +
> +	/* Enable uprobes in the system */
> +	if (tracefs_event_enable(instance, utrace->usystem, NULL)) {
> +		PyErr_SetString(TRACECRUNCHER_ERROR,
> +				"Failed to enable trace events");
> +		return -1;
> +	}
> +
> +	return 0;
> +}
> +
> +#define PERF_EXEC_SYNC	"/TC_PERF_SYNC_XXXXXX"
> +static int utrace_exec_cmd(struct py_utrace_context *utrace, struct tracefs_instance *instance)
> +{
> +	char *envp[] = {NULL};
> +	char sname[strlen(PERF_EXEC_SYNC) + 1];
> +	sem_t *sem;
> +	pid_t pid;
> +	int ret;
> +
> +	strcpy(sname, PERF_EXEC_SYNC);
> +	mktemp(sname);
> +	sem = sem_open(sname, O_CREAT | O_EXCL, 0644, 0);
> +	sem_unlink(sname);
> +
> +	pid = fork();
> +	if (pid < 0) {
> +		PyErr_SetString(TRACECRUNCHER_ERROR, "Failed to fork");
> +		return -1;
> +	}
> +	if (pid == 0) {
> +		sem_wait(sem);
> +		execvpe(utrace->fname, utrace->argv, envp);
> +	} else {
> +		utrace->pid = pid;
> +		start_trace(utrace, instance);
> +		sem_post(sem);
> +		return ret;
> +	}
> +
> +	return 0;
> +}
> +
> +static int py_utrace_enable(struct py_utrace_context *utrace, struct tracefs_instance *instance)
> +{
> +	/* If uprobes on desired user functions are not yet generated, do it now */
> +	if (!utrace->uevents.count)
> +		py_utrace_generate_uprobes(utrace);
> +
> +	/* No functions are found in the given program / pid */
> +	if (!utrace->uevents.count) {
> +		PyErr_SetString(TRACECRUNCHER_ERROR,
> +				"Cannot find requested user functions");
> +		return -1;
> +	}
> +
> +	if (utrace->fname)
> +		utrace_exec_cmd(utrace, instance);
> +	else
> +		start_trace(utrace, instance);
> +
> +	return 0;
> +}
> +
> +static int py_utrace_disable(struct py_utrace_context *utrace, struct tracefs_instance *instance)
> +{
> +	/* Disable uprobes in the system */
> +	if (tracefs_event_disable(instance, utrace->usystem, NULL)) {
> +		PyErr_SetString(TRACECRUNCHER_ERROR,
> +				"Failed to disable trace events");
> +		return -1;
> +	}
> +
> +	return 0;
> +}
> +
> +static bool tracing_run;
> +
> +static void tracing_stop(int sig)
> +{
> +	tracing_run = false;
> +}
> +
> +static void tracing_timer(int sig, siginfo_t *si, void *uc)
> +{
> +	tracing_run = false;
> +}
> +
> +#define PID_WAIT_CHECK_USEC	500000
> +#define TIMER_SEC_NANO		1000000000LL
> +static int utrace_wait_pid(struct py_utrace_context *utrace)
> +{
> +	struct itimerspec tperiod = {0};
> +	struct sigaction saction = {0};
> +	struct sigevent stime = {0};
> +	timer_t timer_id;
> +
> +	if (utrace->pid == 0)
> +		return -1;
> +
> +	tracing_run = true;
> +	signal(SIGINT, tracing_stop);
> +
> +	if (utrace->trace_time) {
> +		stime.sigev_notify = SIGEV_SIGNAL;
> +		stime.sigev_signo = SIGRTMIN;
> +		if (timer_create(CLOCK_MONOTONIC, &stime, &timer_id))
> +			return -1;
> +		saction.sa_flags = SA_SIGINFO;
> +		saction.sa_sigaction = tracing_timer;
> +		sigemptyset(&saction.sa_mask);
> +		if (sigaction(SIGRTMIN, &saction, NULL)) {
> +			timer_delete(timer_id);
> +			return -1;
> +		}
> +		/* covert trace_time from msec to sec, nsec */
> +		tperiod.it_value.tv_nsec = ((unsigned long long)utrace->trace_time * 1000000LL);
> +		if (tperiod.it_value.tv_nsec >= TIMER_SEC_NANO) {
> +			tperiod.it_value.tv_sec = tperiod.it_value.tv_nsec / TIMER_SEC_NANO;
> +			tperiod.it_value.tv_nsec %= TIMER_SEC_NANO;
> +		}
> +		if (timer_settime(timer_id, 0, &tperiod, NULL))
> +			return -1;
> +	}
> +
> +	do {
> +		if (utrace->fname) { /* wait for a child */
> +			if (waitpid(utrace->pid, NULL, WNOHANG) == (int)utrace->pid) {
> +				utrace->pid = 0;
> +				tracing_run = false;
> +			}
> +		} else { /* not a child, check if still exist */
> +			if (kill(utrace->pid, 0) == -1 && errno == ESRCH) {
> +				utrace->pid = 0;
> +				tracing_run = false;
> +			}
> +		}
> +		usleep(PID_WAIT_CHECK_USEC);
> +	} while (tracing_run);
> +
> +	if (utrace->trace_time)
> +		timer_delete(timer_id);
> +
> +	signal(SIGINT, SIG_DFL);
> +
> +	return 0;
> +}
> +
> +PyObject *PyUserTrace_enable(PyUserTrace *self, PyObject *args, PyObject *kwargs)
> +{
> +	struct py_utrace_context *utrace = self->ptrObj;
> +	static char *kwlist[] = {"instance", "wait", "time", NULL};
> +	struct tracefs_instance *instance = NULL;
> +	PyObject *py_inst = NULL;
> +	int wait = false;
> +	int ret;
> +
> +	if (!utrace) {
> +		PyErr_SetString(TRACECRUNCHER_ERROR,
> +				"Failed to get utrace context");
> +		return NULL;
> +	}
> +
> +	if (!PyArg_ParseTupleAndKeywords(args,
> +					 kwargs,
> +					 "|Ops",
> +					 kwlist,
> +					 &py_inst,
> +					 &wait,
> +					 &utrace->trace_time)) {
> +		PyErr_SetString(TRACECRUNCHER_ERROR,
> +				"Failed to parse input arguments");
> +		return NULL;
> +	}
> +
> +	if (!get_optional_instance(py_inst, &instance))
> +		return NULL;
> +
> +	ret = py_utrace_enable(utrace, instance);
> +	if (ret)
> +		return NULL;
> +
> +	if (wait) {
> +		utrace_wait_pid(utrace);
> +		py_utrace_disable(utrace, instance);
> +	}
> +
> +	Py_RETURN_NONE;
> +}
> +
> +PyObject *PyUserTrace_disable(PyUserTrace *self, PyObject *args, PyObject *kwargs)
> +{
> +	struct py_utrace_context *utrace = self->ptrObj;
> +	static char *kwlist[] = {"instance", NULL};
> +	struct tracefs_instance *instance = NULL;
> +	PyObject *py_inst = NULL;
> +	int ret;
> +
> +	if (!utrace) {
> +		PyErr_SetString(TRACECRUNCHER_ERROR,
> +				"Failed to get utrace context");
> +		return NULL;
> +	}
> +
> +	if (!PyArg_ParseTupleAndKeywords(args,
> +					 kwargs,
> +					 "|O",
> +					 kwlist,
> +					 &py_inst)) {
> +		PyErr_SetString(TRACECRUNCHER_ERROR,
> +				"Failed to parse input arguments");
> +		return NULL;
> +	}
> +
> +	if (!get_optional_instance(py_inst, &instance))
> +		return NULL;
> +
> +	ret = py_utrace_disable(utrace, instance);
> +
> +	if (ret)
> +		return NULL;
> +
> +	Py_RETURN_NONE;
> +}
> +
> +PyObject *PyFtrace_utrace(PyObject *self, PyObject *args, PyObject *kwargs)
This must be PyFtrace_user_trace()

> +{
> +	static char *kwlist[] = {"pid", "name", "follow_libs", "argv", NULL};

I still think the 'name' argument is redundant. It is the same as argv[0].

Note that when we use 'execvPe()', there is no need to provide the absolute path. All locations provided in $PATH will 
be searched, that is identical to calling 'shutil.which()' in the user code.

Even if we need the absolute path for something else, this path can be derived internally.


> +	struct py_utrace_context *utrace;
> +	char **argv = NULL;
> +	long long pid = 0;
> +	char *comm = NULL;
> +	int libs = 0;
> +	PyObject *py_utrace, *py_arg, *py_args = NULL;
> +	int i, argc;
> +
> +	if (!PyArg_ParseTupleAndKeywords(args,
> +					 kwargs,
> +					 "|KspO",
> +					 kwlist,
> +					 &pid,
> +					 &comm,
> +					 &libs,
> +					 &py_args)) {
> +		return NULL;
> +	}
> +
> +	if (pid == 0 && !comm) {
If both 'pid' and 'name' (argv) are provided, this must be considered error (not silently taking one of the two) and a 
proper error message must be printed. Also error in the case when 'pid' is negative.

Thanks!
Yordan

> +		PyErr_Format(TFS_ERROR,
> +			     "Process ID or program name should be specified");
> +		return NULL;
> +	}
> +
> +	if (comm && py_args) {
> +		if (!PyList_CheckExact(py_args)) {
> +			PyErr_SetString(TFS_ERROR, "Failed to parse argv list");
> +			return NULL;
> +		}
> +		argc = PyList_Size(py_args);
> +		argv = calloc(argc + 1, sizeof(char *));
> +		for (i = 0; i < argc; i++) {
> +			py_arg = PyList_GetItem(py_args, i);
> +			if (!PyUnicode_Check(py_arg))
> +				continue;
> +			argv[i] = strdup(PyUnicode_DATA(py_arg));
> +			if (!argv[i])
> +				goto error;
> +		}
> +		argv[i] = NULL;
> +	}
> +
> +	utrace = utrace_new(pid, comm, argv, libs);
> +	if (!utrace) {
> +		MEM_ERROR;
> +		goto error;
> +	}
> +	py_utrace = PyUserTrace_New(utrace);
> +
> +	return py_utrace;
> +
> +error:
> +	if (argv) {
> +		for (i = 0; i < argc; i++)
> +			free(argv[i]);
> +		free(argv);
> +	}
> +	return NULL;
> +}
> diff --git a/src/ftracepy-utils.h b/src/ftracepy-utils.h
> index e6fab69..0b5000b 100644
> --- a/src/ftracepy-utils.h
> +++ b/src/ftracepy-utils.h
> @@ -34,6 +34,21 @@ C_OBJECT_WRAPPER_DECLARE(tracefs_synth, PySynthEvent)
>   
>   PyObject *PyTepRecord_time(PyTepRecord* self);
>   
> +struct py_utrace_context;
> +void py_utrace_free(struct py_utrace_context *utrace);
> +int py_utrace_destroy(struct py_utrace_context *utrace);
> +C_OBJECT_WRAPPER_DECLARE(py_utrace_context, PyUserTrace);
> +
> +PyObject *PyUserTrace_add_function(PyUserTrace *self, PyObject *args,
> +				   PyObject *kwargs);
> +
> +PyObject *PyUserTrace_add_ret_function(PyUserTrace *self, PyObject *args,
> +				       PyObject *kwargs);
> +
> +PyObject *PyUserTrace_enable(PyUserTrace *self, PyObject *args, PyObject *kwargs);
> +
> +PyObject *PyUserTrace_disable(PyUserTrace *self, PyObject *args, PyObject *kwargs);
> +
>   PyObject *PyTepRecord_cpu(PyTepRecord* self);
>   
>   PyObject *PyTepEvent_name(PyTepEvent* self);
> @@ -270,6 +285,8 @@ PyObject *PyFtrace_synth(PyObject *self, PyObject *args,
>   PyObject *PyFtrace_set_ftrace_loglevel(PyObject *self, PyObject *args,
>   						       PyObject *kwargs);
>   
> +PyObject *PyFtrace_utrace(PyObject *self, PyObject *args, PyObject *kwargs);
> +
>   PyObject *PyFtrace_trace_process(PyObject *self, PyObject *args,
>   						 PyObject *kwargs);
>   
> diff --git a/src/ftracepy.c b/src/ftracepy.c
> index 681d641..6ca9df0 100644
> --- a/src/ftracepy.c
> +++ b/src/ftracepy.c
> @@ -315,6 +315,32 @@ C_OBJECT_WRAPPER(tracefs_synth, PySynthEvent,
>   		 tracefs_synth_destroy,
>   		 tracefs_synth_free)
>   
> +static PyMethodDef PyUserTrace_methods[] = {
> +	{"add_function",
> +	 (PyCFunction) PyUserTrace_add_function,
> +	 METH_VARARGS | METH_KEYWORDS,
> +	 "Add tracepoint on user function."
> +	},
> +	{"add_ret_function",
> +	 (PyCFunction) PyUserTrace_add_ret_function,
> +	 METH_VARARGS | METH_KEYWORDS,
> +	 "Add tracepoint on user function return."
> +	},
> +	{"enable",
> +	 (PyCFunction) PyUserTrace_enable,
> +	 METH_VARARGS | METH_KEYWORDS,
> +	 "Enable tracing of the configured user tracepoints."
> +	},
> +	{"disable",
> +	 (PyCFunction) PyUserTrace_disable,
> +	 METH_VARARGS | METH_KEYWORDS,
> +	 "Disable tracing of the configured user tracepoints."
> +	},
> +	{NULL, NULL, 0, NULL}
> +};
> +C_OBJECT_WRAPPER(py_utrace_context, PyUserTrace,
> +		 py_utrace_destroy, py_utrace_free)
> +
>   static PyMethodDef ftracepy_methods[] = {
>   	{"dir",
>   	 (PyCFunction) PyFtrace_dir,
> @@ -501,6 +527,11 @@ static PyMethodDef ftracepy_methods[] = {
>   	 METH_VARARGS | METH_KEYWORDS,
>   	 "Define a synthetic event."
>   	},
> +	{"user_trace",
> +	 (PyCFunction) PyFtrace_utrace,
> +	 METH_VARARGS | METH_KEYWORDS,
> +	 "Create a context for tracing a user process using uprobes"
> +	},
>   	{"set_ftrace_loglevel",
>   	 (PyCFunction) PyFtrace_set_ftrace_loglevel,
>   	 METH_VARARGS | METH_KEYWORDS,
> @@ -575,6 +606,9 @@ PyMODINIT_FUNC PyInit_ftracepy(void)
>   	if (!PySynthEventTypeInit())
>   		return NULL;
>   
> +	if (!PyUserTraceTypeInit())
> +		return NULL;
> +
>   	TFS_ERROR = PyErr_NewException("tracecruncher.ftracepy.tfs_error",
>   				       NULL, NULL);
>   
> @@ -593,6 +627,7 @@ PyMODINIT_FUNC PyInit_ftracepy(void)
>   	PyModule_AddObject(module, "tracefs_dynevent", (PyObject *) &PyDyneventType);
>   	PyModule_AddObject(module, "tracefs_hist", (PyObject *) &PyTraceHistType);
>   	PyModule_AddObject(module, "tracefs_synth", (PyObject *) &PySynthEventType);
> +	PyModule_AddObject(module, "py_utrace_context", (PyObject *) &PyUserTraceType);
>   
>   	PyModule_AddObject(module, "tfs_error", TFS_ERROR);
>   	PyModule_AddObject(module, "tep_error", TEP_ERROR);

Re: [RFC PATCH v2 4/4] trace-cruncher: Example script for uprobes high level API

[Yordan Karadzhov]

On 15.04.22 г. 7:10 ч., Tzvetomir Stoyanov (VMware) wrote:
> Proposed example illustrates how to use uprobes high level API to trace
> all functions for user program. It can be attached to already running
> program, or run the program and trace its execution.
> 
> Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
> ---
>   examples/user_trace.py | 43 ++++++++++++++++++++++++++++++++++++++++++
>   1 file changed, 43 insertions(+)
>   create mode 100755 examples/user_trace.py
> 
> diff --git a/examples/user_trace.py b/examples/user_trace.py
> new file mode 100755
> index 0000000..640ace8
> --- /dev/null
> +++ b/examples/user_trace.py
> @@ -0,0 +1,43 @@
> +#!/usr/bin/env python3
> +
> +"""
> +SPDX-License-Identifier: CC-BY-4.0
> +
> +Copyright 2022 VMware Inc, Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
> +"""
> +
> +import sys
> +import shutil
> +
> +import tracecruncher.ftracepy as ft
> +
> +if __name__ == "__main__":
> +    if len(sys.argv) < 2:
> +        print('Usage: ', sys.argv[0], ' [PROCESS]')
         print('Usage: ', sys.argv[0], ' [PROCESS or PID]')
> +        sys.exit(1)
> +
> +    # Create new Ftrace instance to work in. The tracing in this new instance
> +    # is not going to be enabled yet.
> +    inst = ft.create_instance(tracing_on=False)
> +
> +    # Create a user tracing context for given process, exclude the libraries
> +    if sys.argv[1].isdigit():
> +        utrace = ft.user_trace(pid=int(sys.argv[1]), follow_libs=False)
> +    else:
> +        file = shutil.which(sys.argv[1])
> +        if file is None:
> +            print('Cannot find ', sys.argv[1], ' in the system')
> +            sys.exit(1)
> +        utrace = ft.user_trace(name=file, argv=sys.argv[1:], follow_libs=False)
> +
> +    # Trace execution of all available functions in the given process
> +    utrace.add_function(fname = "*")
No need for spaces around '='.

> +
> +    # Add trace points on functions return as well
> +    utrace.add_ret_function(fname = "*")
Same here

> +
> +    # Start tracing in an instance
> +    utrace.enable(instance = inst, wait=False)
and here.
> +
> +    # Read the trace buffer during the trace until ctrl-c is pressed
> +    ft.read_trace(instance=inst)
> \ No newline at end of file
Add new line at the end.

Re: [RFC PATCH v2 3/4] trace-cruncher: High level wrappers for ftrace uprobes

[Tzvetomir Stoyanov]

On Mon, Apr 18, 2022 at 2:54 PM Yordan Karadzhov <y.karadz@gmail.com> wrote:
>
>
>
> On 15.04.22 г. 7:10 ч., Tzvetomir Stoyanov (VMware) wrote:
> > Using uprobes requires finding the offset of a user function within the
> > binary file, where this functions is compiled. This is not a trivial
> > task, especially in the cases when a bunch of uprobes to user functions
> > should be added.
> > A high level trace-cruncher API allows adding multiple user functions as
> > uprobes or uretprobes. It supports wildcards for function names and
> > adding uprobes for library functions, used by the applications.
> >
> > Signed-off-by: Tzvetomir Stoyanov (VMware) <tz.stoyanov@gmail.com>
> > ---
> >   setup.py             |   4 +-
> >   src/ftracepy-utils.c | 628 +++++++++++++++++++++++++++++++++++++++++++
> >   src/ftracepy-utils.h |  17 ++
> >   src/ftracepy.c       |  35 +++
> >   4 files changed, 682 insertions(+), 2 deletions(-)
> >
> > diff --git a/setup.py b/setup.py
> > index 58561cf..464d3d2 100644
> > --- a/setup.py
> > +++ b/setup.py
> > @@ -71,8 +71,8 @@ def extension(name, sources, libraries):
> >
> >   def main():
> >       module_ft = extension(name='tracecruncher.ftracepy',
> > -                          sources=['src/ftracepy.c', 'src/ftracepy-utils.c'],
> > -                          libraries=['traceevent', 'tracefs'])
> > +                          sources=['src/ftracepy.c', 'src/ftracepy-utils.c', 'src/trace-obj-debug.c'],
> > +                          libraries=['traceevent', 'tracefs', 'bfd'])
> >
> >       cythonize('src/npdatawrapper.pyx', language_level = 3)
> >       module_data = extension(name='tracecruncher.npdatawrapper',
> > diff --git a/src/ftracepy-utils.c b/src/ftracepy-utils.c
> > index b39459b..d420dce 100644
> > --- a/src/ftracepy-utils.c
> > +++ b/src/ftracepy-utils.c
> > @@ -15,15 +15,46 @@
> >   #include <sys/utsname.h>
> >   #include <sys/wait.h>
> >   #include <signal.h>
> > +#include <semaphore.h>
> >   #include <time.h>
> >
> >   // trace-cruncher
> >   #include "ftracepy-utils.h"
> > +#include "trace-obj-debug.h"
> >
> >   PyObject *TFS_ERROR;
> >   PyObject *TEP_ERROR;
> >   PyObject *TRACECRUNCHER_ERROR;
> >
> > +#define UPROBES_SYSTEM "tc_uprobes"
>
> Note that we already have
>
> #define TC_SYS  "tcrunch"
>
> Does it make sence to define a second trace-cruncher system?

Yes, actually this is a template of the system name, the real name is
"tc_uprobes_<pid>" or "tc_uprobes_<file>". I can use TC_SYS as
template, instead of adding another define, but the system name should
be unique for current uprobes trace session, as it is used to enable /
disable all dynamic events related to this session at once.

>
> > +
> > +#define FTRACE_UPROBE                0x1
> > +#define FTRACE_URETPROBE     0x2
> > +
> > +struct fprobes_list {
> > +     int size;
> > +     int count;
> > +     void **data;
> > +};
> > +
> > +struct utrace_func {
> > +     int type;
> > +     char *func_name;
> > +     char *func_args;
> > +};
> > +
> > +struct py_utrace_context {
> > +     pid_t pid;
> > +     char *fname;
> > +     char **argv;
> > +     char *usystem;
> > +     uint32_t trace_time; /* in msec */
> > +     struct fprobes_list fretprobes;
> > +     struct fprobes_list ufuncs;
> > +     struct fprobes_list uevents;
> > +     struct dbg_trace_context *dbg;
> > +};
> > +
> >   static char *kernel_version()
> >   {
> >       struct utsname uts;
> > @@ -3508,3 +3539,600 @@ void PyFtrace_at_exit(void)
> >       if (seq.buffer)
> >               trace_seq_destroy(&seq);
> >   }
> > +
> > +#define INIT_SIZE    100
> This name is too general. Try to make it clear what is to be initialized with this size.
>
> > +static int utrace_list_add(struct fprobes_list *list, void *data)
> > +{
> > +     void **tmp;
> > +     int size;
> > +
> > +     if (list->size <= list->count) {
> > +             if (!list->size)
> > +                     size = INIT_SIZE;
> > +             else
> > +                     size = 2 * list->size;
> > +             tmp = realloc(list->data, size * sizeof(void *));
> > +             if (!tmp)
> > +                     return -1;
> > +             list->data = tmp;
> > +             list->size = size;
> > +     }
> > +
> > +     list->data[list->count] = data;
> > +     list->count++;
> > +     return list->count - 1;
> > +}
> > +
> > +void py_utrace_free(struct py_utrace_context *utrace)
> > +{
> > +     struct utrace_func *f;
> > +     int i;
> > +
> > +     if (!utrace)
> > +             return;
> > +     if (utrace->dbg)
> > +             dbg_trace_context_destroy(utrace->dbg);
> > +
> > +     for (i = 0; i < utrace->ufuncs.count; i++) {
> > +             f = utrace->ufuncs.data[i];
> > +             free(f->func_name);
> > +             free(f);
> > +     }
> > +     free(utrace->ufuncs.data);
> > +     if (utrace->argv) {
> > +             i = 0;
> > +             while (utrace->argv[i])
> > +                     free(utrace->argv[i++]);
> > +             free(utrace->argv);
> > +     }
> > +
> > +     for (i = 0; i < utrace->uevents.count; i++)
> > +             tracefs_dynevent_free(utrace->uevents.data[i]);
> > +     free(utrace->uevents.data);
> > +
> > +     free(utrace->fname);
> > +     free(utrace->usystem);
> > +     free(utrace);
> > +}
> > +
> > +/*
> > + * All strings, used as ftrace system or event name must contain only
> > + * alphabetic characters, digits or underscores.
> > + */
> > +static void fname_unify(char *fname)
> > +{
> > +     int i;
> > +
> > +     for (i = 0; fname[i]; i++)
> > +             if (!isalnum(fname[i]) && fname[i] != '_')
> > +                     fname[i] = '_';
> > +}
> > +
> > +int py_utrace_destroy(struct py_utrace_context *utrace)
> > +{
> > +     int i;
> > +
> > +     for (i = 0; i < utrace->uevents.count; i++)
> > +             tracefs_dynevent_destroy(utrace->uevents.data[i], true);
> > +
> > +     return 0;
> > +}
> > +
> > +static struct py_utrace_context *utrace_new(pid_t pid, char *fname, char **argv, bool libs)
> > +{
> > +     struct py_utrace_context *utrace;
> > +     char *file;
> > +
> > +     utrace = calloc(1, sizeof(*utrace));
> > +     if (!utrace)
> > +             return NULL;
> > +
> > +     if (fname) {
> > +             utrace->argv = argv;
> > +             utrace->dbg = dbg_trace_context_create_file(fname, libs);
> > +             if (!utrace->dbg)
> > +                     goto error;
> > +             utrace->fname = strdup(fname);
> > +             if (!utrace->fname)
> > +                     goto error;
> > +             file = strrchr(fname, '/');
> > +             if (file)
> > +                     file++;
> > +             if (!file || *file == '\0')
> > +                     file = fname;
> > +             if (asprintf(&utrace->usystem, "%s_%s", UPROBES_SYSTEM, file) <= 0)
> > +                     goto error;
> > +     } else {
> > +             utrace->pid = pid;
> > +             utrace->dbg = dbg_trace_context_create_pid(pid, libs);
> > +             if (!utrace->dbg)
> > +                     goto error;
> > +             if (asprintf(&utrace->usystem, "%s_%d", UPROBES_SYSTEM, pid) <= 0)
> > +                     goto error;
> > +     }
> > +
> > +     fname_unify(utrace->usystem);
> > +     return utrace;
> > +
> > +error:
> > +     py_utrace_free(utrace);
> > +     return NULL;
> > +}
> > +
> > +static int py_utrace_add_func(struct py_utrace_context *utrace, char *func, int type)
> > +{
> > +     struct utrace_func *p;
> > +     int ret;
> > +     int i;
> > +
> > +     for (i = 0; i < utrace->ufuncs.count; i++) {
> > +             p = utrace->ufuncs.data[i];
> > +             if (!strcmp(p->func_name, func)) {
> > +                     p->type |= type;
> > +                     return 0;
> > +             }
> > +     }
> > +
> > +     p = calloc(1, sizeof(*p));
> > +     if (!p)
> > +             return -1;
> > +     p->func_name = strdup(func);
> > +     if (!p->func_name)
> > +             goto error;
> > +     p->type = type;
> > +
> > +     ret = utrace_list_add(&utrace->ufuncs, p);
> > +     if (ret < 0)
> > +             goto error;
> > +
> > +     if (dbg_trace_add_resolve_symbol(utrace->dbg, 0, func, ret))
> > +             goto error;
> > +
> > +     return 0;
> > +
> > +error:
> > +     free(p->func_name);
> > +     free(p);
> > +     return -1;
> > +}
> > +
> > +PyObject *PyUserTrace_add_function(PyUserTrace *self, PyObject *args,
> > +                                PyObject *kwargs)
> > +{
> > +     struct py_utrace_context *utrace = self->ptrObj;
> > +     static char *kwlist[] = {"fname", NULL};
> > +     char *fname;
> > +
> > +     if (!PyArg_ParseTupleAndKeywords(args,
> > +                                      kwargs,
> > +                                      "s",
> > +                                      kwlist,
> > +                                      &fname)) {
> > +             return NULL;
> > +     }
> > +
> > +     if (py_utrace_add_func(utrace, fname, FTRACE_UPROBE) < 0) {
> > +             MEM_ERROR
> > +             return NULL;
> > +     }
> > +
> > +     Py_RETURN_NONE;
> > +}
> > +
> > +PyObject *PyUserTrace_add_ret_function(PyUserTrace *self, PyObject *args,
> > +                                    PyObject *kwargs)
> > +{
> > +     struct py_utrace_context *utrace = self->ptrObj;
> > +     static char *kwlist[] = {"fname", NULL};
> > +     char *fname;
> > +
> > +     if (!PyArg_ParseTupleAndKeywords(args,
> > +                                      kwargs,
> > +                                      "s",
> > +                                      kwlist,
> > +                                      &fname)) {
> > +             return NULL;
> > +     }
> > +
> > +     if (py_utrace_add_func(utrace, fname, FTRACE_URETPROBE) < 0) {
> > +             MEM_ERROR
> > +             return NULL;
> > +     }
> > +
> > +     Py_RETURN_NONE;
> > +}
> > +
> > +/*
> > + * max event name is 64 bytes, hard coded in the kernel.
> > + * it can consists only of alphabetic characters, digits or underscores
> > + */
> Please start comments with capital and end with period.
>
>
> > +#define FILENAME_TRUNCATE    10
> > +#define FUNCAME_TRUNCATE     50
> > +static char *uprobe_event_name(char *file, char *func, int type)
> > +{
> > +     char *event = NULL;
> > +     char *fname;
> > +
> > +     fname = strrchr(file, '/');
> > +     if (fname)
> > +             fname++;
> > +     if (!fname || *fname == '\0')
> > +             fname = file;
> > +
> > +     asprintf(&event, "%s%.*s_%.*s",
> > +              type == FTRACE_URETPROBE ? "r_":"",
> > +              FILENAME_TRUNCATE, fname, FUNCAME_TRUNCATE, func);
> > +     if (event)
> > +             fname_unify(event);
> > +
> > +     return event;
> > +}
> > +
> > +/*
> > + * Create uprobe based on function name,
> > + * file name and function offset within the file
> > + */
> > +static int utrace_event_create(struct py_utrace_context *utrace,
> > +                            struct dbg_trace_symbols *sym, char *fecthargs,
> > +                            int type)
> > +{
> > +     struct tracefs_dynevent *uevent = NULL;
> > +     char *rname;
> > +
> > +     /* Generate uprobe event name, according to ftrace name requirements */
> > +     rname = uprobe_event_name(sym->fname, sym->name, type);
> > +     if (!rname)
> > +             return -1;
> > +
> > +     if (type == FTRACE_URETPROBE)
> > +             uevent = tracefs_uretprobe_alloc(utrace->usystem, rname,
> > +                                              sym->fname, sym->foffset, fecthargs);
> > +     else
> > +             uevent = tracefs_uprobe_alloc(utrace->usystem, rname,
> > +                                           sym->fname, sym->foffset, fecthargs);
> > +
> > +     free(rname);
> > +     if (!uevent)
> > +             return -1;
> > +
> > +     if (tracefs_dynevent_create(uevent)) {
> > +             tracefs_dynevent_free(uevent);
> > +             return -1;
> > +     }
> > +
> > +     utrace_list_add(&utrace->uevents, uevent);
> > +     return 0;
> > +}
> > +
> > +/* callback, called on each resolved function */
> > +static int symblos_walk(struct dbg_trace_symbols *sym, void *context)
> > +{
> > +     struct py_utrace_context *utrace = context;
> > +     struct utrace_func *ufunc;
> > +
> > +     if (!sym->name || !sym->fname || !sym->foffset ||
> > +         sym->cookie < 0 || sym->cookie >= utrace->ufuncs.count)
> > +             return 0;
> > +
> > +     ufunc = utrace->ufuncs.data[sym->cookie];
> > +
> > +     if (ufunc->type & FTRACE_UPROBE)
> > +             utrace_event_create(utrace, sym, ufunc->func_args, FTRACE_UPROBE);
> > +
> > +     if (ufunc->type & FTRACE_URETPROBE)
> > +             utrace_event_create(utrace, sym, ufunc->func_args, FTRACE_URETPROBE);
> > +
> > +     return 0;
> > +}
> > +
> > +static void py_utrace_generate_uprobes(struct py_utrace_context *utrace)
> > +{
> > +     /* Find the exact name and file offset of each user function that should be traced */
> > +     dbg_trace_resolve_symbols(utrace->dbg);
> > +     dbg_trace_walk_resolved_symbols(utrace->dbg, symblos_walk, utrace);
> > +}
> > +
> > +static int start_trace(struct py_utrace_context *utrace, struct tracefs_instance *instance)
>
> The name is too general.

As it was a static function in a uprobe specific file, I put a general
name. But when I moved the code to ftrace-utils.c file, didn't realize
that the code there is not uprobe specific. I'll change it to a more
unique name in the next version.

>
> > +{
> > +     PyObject *pPid = PyLong_FromLong(utrace->pid);
> > +     bool ret;
> > +
> > +     if (!pPid)
> > +             return -1;
> > +
> > +     /* Filter the trace only on desired pid(s) */
> > +     ret = hook2pid(instance, PyLong_FromLong(utrace->pid), true);
> > +     Py_DECREF(pPid);
> > +     if (!ret) {
> > +             PyErr_SetString(TRACECRUNCHER_ERROR,
> > +                             "Failed to set trace filter");
> > +             return -1;
> > +     }
> > +
> > +     /* Enable uprobes in the system */
> > +     if (tracefs_event_enable(instance, utrace->usystem, NULL)) {
> > +             PyErr_SetString(TRACECRUNCHER_ERROR,
> > +                             "Failed to enable trace events");
> > +             return -1;
> > +     }
> > +
> > +     return 0;
> > +}
> > +
> > +#define PERF_EXEC_SYNC       "/TC_PERF_SYNC_XXXXXX"
> > +static int utrace_exec_cmd(struct py_utrace_context *utrace, struct tracefs_instance *instance)
> > +{
> > +     char *envp[] = {NULL};
> > +     char sname[strlen(PERF_EXEC_SYNC) + 1];
> > +     sem_t *sem;
> > +     pid_t pid;
> > +     int ret;
> > +
> > +     strcpy(sname, PERF_EXEC_SYNC);
> > +     mktemp(sname);
> > +     sem = sem_open(sname, O_CREAT | O_EXCL, 0644, 0);
> > +     sem_unlink(sname);
> > +
> > +     pid = fork();
> > +     if (pid < 0) {
> > +             PyErr_SetString(TRACECRUNCHER_ERROR, "Failed to fork");
> > +             return -1;
> > +     }
> > +     if (pid == 0) {
> > +             sem_wait(sem);
> > +             execvpe(utrace->fname, utrace->argv, envp);
> > +     } else {
> > +             utrace->pid = pid;
> > +             start_trace(utrace, instance);
> > +             sem_post(sem);
> > +             return ret;
> > +     }
> > +
> > +     return 0;
> > +}
> > +
> > +static int py_utrace_enable(struct py_utrace_context *utrace, struct tracefs_instance *instance)
> > +{
> > +     /* If uprobes on desired user functions are not yet generated, do it now */
> > +     if (!utrace->uevents.count)
> > +             py_utrace_generate_uprobes(utrace);
> > +
> > +     /* No functions are found in the given program / pid */
> > +     if (!utrace->uevents.count) {
> > +             PyErr_SetString(TRACECRUNCHER_ERROR,
> > +                             "Cannot find requested user functions");
> > +             return -1;
> > +     }
> > +
> > +     if (utrace->fname)
> > +             utrace_exec_cmd(utrace, instance);
> > +     else
> > +             start_trace(utrace, instance);
> > +
> > +     return 0;
> > +}
> > +
> > +static int py_utrace_disable(struct py_utrace_context *utrace, struct tracefs_instance *instance)
> > +{
> > +     /* Disable uprobes in the system */
> > +     if (tracefs_event_disable(instance, utrace->usystem, NULL)) {
> > +             PyErr_SetString(TRACECRUNCHER_ERROR,
> > +                             "Failed to disable trace events");
> > +             return -1;
> > +     }
> > +
> > +     return 0;
> > +}
> > +
> > +static bool tracing_run;
> > +
> > +static void tracing_stop(int sig)
> > +{
> > +     tracing_run = false;
> > +}
> > +
> > +static void tracing_timer(int sig, siginfo_t *si, void *uc)
> > +{
> > +     tracing_run = false;
> > +}
> > +
> > +#define PID_WAIT_CHECK_USEC  500000
> > +#define TIMER_SEC_NANO               1000000000LL
> > +static int utrace_wait_pid(struct py_utrace_context *utrace)
> > +{
> > +     struct itimerspec tperiod = {0};
> > +     struct sigaction saction = {0};
> > +     struct sigevent stime = {0};
> > +     timer_t timer_id;
> > +
> > +     if (utrace->pid == 0)
> > +             return -1;
> > +
> > +     tracing_run = true;
> > +     signal(SIGINT, tracing_stop);
> > +
> > +     if (utrace->trace_time) {
> > +             stime.sigev_notify = SIGEV_SIGNAL;
> > +             stime.sigev_signo = SIGRTMIN;
> > +             if (timer_create(CLOCK_MONOTONIC, &stime, &timer_id))
> > +                     return -1;
> > +             saction.sa_flags = SA_SIGINFO;
> > +             saction.sa_sigaction = tracing_timer;
> > +             sigemptyset(&saction.sa_mask);
> > +             if (sigaction(SIGRTMIN, &saction, NULL)) {
> > +                     timer_delete(timer_id);
> > +                     return -1;
> > +             }
> > +             /* covert trace_time from msec to sec, nsec */
> > +             tperiod.it_value.tv_nsec = ((unsigned long long)utrace->trace_time * 1000000LL);
> > +             if (tperiod.it_value.tv_nsec >= TIMER_SEC_NANO) {
> > +                     tperiod.it_value.tv_sec = tperiod.it_value.tv_nsec / TIMER_SEC_NANO;
> > +                     tperiod.it_value.tv_nsec %= TIMER_SEC_NANO;
> > +             }
> > +             if (timer_settime(timer_id, 0, &tperiod, NULL))
> > +                     return -1;
> > +     }
> > +
> > +     do {
> > +             if (utrace->fname) { /* wait for a child */
> > +                     if (waitpid(utrace->pid, NULL, WNOHANG) == (int)utrace->pid) {
> > +                             utrace->pid = 0;
> > +                             tracing_run = false;
> > +                     }
> > +             } else { /* not a child, check if still exist */
> > +                     if (kill(utrace->pid, 0) == -1 && errno == ESRCH) {
> > +                             utrace->pid = 0;
> > +                             tracing_run = false;
> > +                     }
> > +             }
> > +             usleep(PID_WAIT_CHECK_USEC);
> > +     } while (tracing_run);
> > +
> > +     if (utrace->trace_time)
> > +             timer_delete(timer_id);
> > +
> > +     signal(SIGINT, SIG_DFL);
> > +
> > +     return 0;
> > +}
> > +
> > +PyObject *PyUserTrace_enable(PyUserTrace *self, PyObject *args, PyObject *kwargs)
> > +{
> > +     struct py_utrace_context *utrace = self->ptrObj;
> > +     static char *kwlist[] = {"instance", "wait", "time", NULL};
> > +     struct tracefs_instance *instance = NULL;
> > +     PyObject *py_inst = NULL;
> > +     int wait = false;
> > +     int ret;
> > +
> > +     if (!utrace) {
> > +             PyErr_SetString(TRACECRUNCHER_ERROR,
> > +                             "Failed to get utrace context");
> > +             return NULL;
> > +     }
> > +
> > +     if (!PyArg_ParseTupleAndKeywords(args,
> > +                                      kwargs,
> > +                                      "|Ops",
> > +                                      kwlist,
> > +                                      &py_inst,
> > +                                      &wait,
> > +                                      &utrace->trace_time)) {
> > +             PyErr_SetString(TRACECRUNCHER_ERROR,
> > +                             "Failed to parse input arguments");
> > +             return NULL;
> > +     }
> > +
> > +     if (!get_optional_instance(py_inst, &instance))
> > +             return NULL;
> > +
> > +     ret = py_utrace_enable(utrace, instance);
> > +     if (ret)
> > +             return NULL;
> > +
> > +     if (wait) {
> > +             utrace_wait_pid(utrace);
> > +             py_utrace_disable(utrace, instance);
> > +     }
> > +
> > +     Py_RETURN_NONE;
> > +}
> > +
> > +PyObject *PyUserTrace_disable(PyUserTrace *self, PyObject *args, PyObject *kwargs)
> > +{
> > +     struct py_utrace_context *utrace = self->ptrObj;
> > +     static char *kwlist[] = {"instance", NULL};
> > +     struct tracefs_instance *instance = NULL;
> > +     PyObject *py_inst = NULL;
> > +     int ret;
> > +
> > +     if (!utrace) {
> > +             PyErr_SetString(TRACECRUNCHER_ERROR,
> > +                             "Failed to get utrace context");
> > +             return NULL;
> > +     }
> > +
> > +     if (!PyArg_ParseTupleAndKeywords(args,
> > +                                      kwargs,
> > +                                      "|O",
> > +                                      kwlist,
> > +                                      &py_inst)) {
> > +             PyErr_SetString(TRACECRUNCHER_ERROR,
> > +                             "Failed to parse input arguments");
> > +             return NULL;
> > +     }
> > +
> > +     if (!get_optional_instance(py_inst, &instance))
> > +             return NULL;
> > +
> > +     ret = py_utrace_disable(utrace, instance);
> > +
> > +     if (ret)
> > +             return NULL;
> > +
> > +     Py_RETURN_NONE;
> > +}
> > +
> > +PyObject *PyFtrace_utrace(PyObject *self, PyObject *args, PyObject *kwargs)
> This must be PyFtrace_user_trace()
>
> > +{
> > +     static char *kwlist[] = {"pid", "name", "follow_libs", "argv", NULL};
>
> I still think the 'name' argument is redundant. It is the same as argv[0].
>
> Note that when we use 'execvPe()', there is no need to provide the absolute path. All locations provided in $PATH will
> be searched, that is identical to calling 'shutil.which()' in the user code.
>
> Even if we need the absolute path for something else, this path can be derived internally.
>

The absolute path is needed for the code that resolves function name
to address and file offset. As there is no elegant way to do that in
C, I did that in the python code. But You are right, the absolute path
is needed by internal logic and should not be part of the API. I'll
look for some hacky way to search the $PATH in the C code.

>
> > +     struct py_utrace_context *utrace;
> > +     char **argv = NULL;
> > +     long long pid = 0;
> > +     char *comm = NULL;
> > +     int libs = 0;
> > +     PyObject *py_utrace, *py_arg, *py_args = NULL;
> > +     int i, argc;
> > +
> > +     if (!PyArg_ParseTupleAndKeywords(args,
> > +                                      kwargs,
> > +                                      "|KspO",
> > +                                      kwlist,
> > +                                      &pid,
> > +                                      &comm,
> > +                                      &libs,
> > +                                      &py_args)) {
> > +             return NULL;
> > +     }
> > +
> > +     if (pid == 0 && !comm) {
> If both 'pid' and 'name' (argv) are provided, this must be considered error (not silently taking one of the two) and a
> proper error message must be printed. Also error in the case when 'pid' is negative.
>
> Thanks!
> Yordan

I'll send the next version addressing your comments.
Thanks!

>
> > +             PyErr_Format(TFS_ERROR,
> > +                          "Process ID or program name should be specified");
> > +             return NULL;
> > +     }
> > +
> > +     if (comm && py_args) {
> > +             if (!PyList_CheckExact(py_args)) {
> > +                     PyErr_SetString(TFS_ERROR, "Failed to parse argv list");
> > +                     return NULL;
> > +             }
> > +             argc = PyList_Size(py_args);
> > +             argv = calloc(argc + 1, sizeof(char *));
> > +             for (i = 0; i < argc; i++) {
> > +                     py_arg = PyList_GetItem(py_args, i);
> > +                     if (!PyUnicode_Check(py_arg))
> > +                             continue;
> > +                     argv[i] = strdup(PyUnicode_DATA(py_arg));
> > +                     if (!argv[i])
> > +                             goto error;
> > +             }
> > +             argv[i] = NULL;
> > +     }
> > +
> > +     utrace = utrace_new(pid, comm, argv, libs);
> > +     if (!utrace) {
> > +             MEM_ERROR;
> > +             goto error;
> > +     }
> > +     py_utrace = PyUserTrace_New(utrace);
> > +
> > +     return py_utrace;
> > +
> > +error:
> > +     if (argv) {
> > +             for (i = 0; i < argc; i++)
> > +                     free(argv[i]);
> > +             free(argv);
> > +     }
> > +     return NULL;
> > +}
> > diff --git a/src/ftracepy-utils.h b/src/ftracepy-utils.h
> > index e6fab69..0b5000b 100644
> > --- a/src/ftracepy-utils.h
> > +++ b/src/ftracepy-utils.h
> > @@ -34,6 +34,21 @@ C_OBJECT_WRAPPER_DECLARE(tracefs_synth, PySynthEvent)
> >
> >   PyObject *PyTepRecord_time(PyTepRecord* self);
> >
> > +struct py_utrace_context;
> > +void py_utrace_free(struct py_utrace_context *utrace);
> > +int py_utrace_destroy(struct py_utrace_context *utrace);
> > +C_OBJECT_WRAPPER_DECLARE(py_utrace_context, PyUserTrace);
> > +
> > +PyObject *PyUserTrace_add_function(PyUserTrace *self, PyObject *args,
> > +                                PyObject *kwargs);
> > +
> > +PyObject *PyUserTrace_add_ret_function(PyUserTrace *self, PyObject *args,
> > +                                    PyObject *kwargs);
> > +
> > +PyObject *PyUserTrace_enable(PyUserTrace *self, PyObject *args, PyObject *kwargs);
> > +
> > +PyObject *PyUserTrace_disable(PyUserTrace *self, PyObject *args, PyObject *kwargs);
> > +
> >   PyObject *PyTepRecord_cpu(PyTepRecord* self);
> >
> >   PyObject *PyTepEvent_name(PyTepEvent* self);
> > @@ -270,6 +285,8 @@ PyObject *PyFtrace_synth(PyObject *self, PyObject *args,
> >   PyObject *PyFtrace_set_ftrace_loglevel(PyObject *self, PyObject *args,
> >                                                      PyObject *kwargs);
> >
> > +PyObject *PyFtrace_utrace(PyObject *self, PyObject *args, PyObject *kwargs);
> > +
> >   PyObject *PyFtrace_trace_process(PyObject *self, PyObject *args,
> >                                                PyObject *kwargs);
> >
> > diff --git a/src/ftracepy.c b/src/ftracepy.c
> > index 681d641..6ca9df0 100644
> > --- a/src/ftracepy.c
> > +++ b/src/ftracepy.c
> > @@ -315,6 +315,32 @@ C_OBJECT_WRAPPER(tracefs_synth, PySynthEvent,
> >                tracefs_synth_destroy,
> >                tracefs_synth_free)
> >
> > +static PyMethodDef PyUserTrace_methods[] = {
> > +     {"add_function",
> > +      (PyCFunction) PyUserTrace_add_function,
> > +      METH_VARARGS | METH_KEYWORDS,
> > +      "Add tracepoint on user function."
> > +     },
> > +     {"add_ret_function",
> > +      (PyCFunction) PyUserTrace_add_ret_function,
> > +      METH_VARARGS | METH_KEYWORDS,
> > +      "Add tracepoint on user function return."
> > +     },
> > +     {"enable",
> > +      (PyCFunction) PyUserTrace_enable,
> > +      METH_VARARGS | METH_KEYWORDS,
> > +      "Enable tracing of the configured user tracepoints."
> > +     },
> > +     {"disable",
> > +      (PyCFunction) PyUserTrace_disable,
> > +      METH_VARARGS | METH_KEYWORDS,
> > +      "Disable tracing of the configured user tracepoints."
> > +     },
> > +     {NULL, NULL, 0, NULL}
> > +};
> > +C_OBJECT_WRAPPER(py_utrace_context, PyUserTrace,
> > +              py_utrace_destroy, py_utrace_free)
> > +
> >   static PyMethodDef ftracepy_methods[] = {
> >       {"dir",
> >        (PyCFunction) PyFtrace_dir,
> > @@ -501,6 +527,11 @@ static PyMethodDef ftracepy_methods[] = {
> >        METH_VARARGS | METH_KEYWORDS,
> >        "Define a synthetic event."
> >       },
> > +     {"user_trace",
> > +      (PyCFunction) PyFtrace_utrace,
> > +      METH_VARARGS | METH_KEYWORDS,
> > +      "Create a context for tracing a user process using uprobes"
> > +     },
> >       {"set_ftrace_loglevel",
> >        (PyCFunction) PyFtrace_set_ftrace_loglevel,
> >        METH_VARARGS | METH_KEYWORDS,
> > @@ -575,6 +606,9 @@ PyMODINIT_FUNC PyInit_ftracepy(void)
> >       if (!PySynthEventTypeInit())
> >               return NULL;
> >
> > +     if (!PyUserTraceTypeInit())
> > +             return NULL;
> > +
> >       TFS_ERROR = PyErr_NewException("tracecruncher.ftracepy.tfs_error",
> >                                      NULL, NULL);
> >
> > @@ -593,6 +627,7 @@ PyMODINIT_FUNC PyInit_ftracepy(void)
> >       PyModule_AddObject(module, "tracefs_dynevent", (PyObject *) &PyDyneventType);
> >       PyModule_AddObject(module, "tracefs_hist", (PyObject *) &PyTraceHistType);
> >       PyModule_AddObject(module, "tracefs_synth", (PyObject *) &PySynthEventType);
> > +     PyModule_AddObject(module, "py_utrace_context", (PyObject *) &PyUserTraceType);
> >
> >       PyModule_AddObject(module, "tfs_error", TFS_ERROR);
> >       PyModule_AddObject(module, "tep_error", TEP_ERROR);



-- 
Tzvetomir (Ceco) Stoyanov
VMware Open Source Technology Center