[PATCH v4 0/7] libtracefs: Facilitate adding and removing kprobes

[PATCH v4 0/7] libtracefs: Facilitate adding and removing kprobes

[Steven Rostedt]

From: "Steven Rostedt (VMware)" <rostedt@goodmis.org>

Add a tracefs_kprobe_raw() to facilitate adding kprobe events. It's
denoted as "raw" because it still requires knowing the complex format of
a kprobe, but at least it helps with other formats:

 p:[[system/]event] addr fmt

The user only needs to know the "fmt" part above, and not worry about
what file to open, or how to open it.

Also add a tracefs_kprobe_clear() to clear all kprobes and a
tracefs_kprobe_clear_probe() to clear an individual kprobe. Both have a
"force" parameter, that if set, will then try to disable the kprobe in
all instances (including the top) and then remove the kprobe(s).

Changes since v3: https://lore.kernel.org/linux-trace-devel/20210702031022.154146-1-rostedt@goodmis.org/

 - Silly me used "stat(KPROBE_EVENTS)" to see if the kprobe event
   file existed, but KPROBE_EVENTS is not a full path. Should have
   used tracefs_file_exists() since that's what it's made for.
   My testing missed this, because I forgot to push the changes to
   my test machine, and I tested the code with old logic.

Steven Rostedt (VMware) (7):
  libtracefs: Implement tracefs_instances()
  libtracefs: Implement tracefs_kprobe_raw()
  libtracefs: Implement tracefs_kretprobe_raw()
  libtracefs: Implement tracefs_get_kprobes()
  libtracefs: Implement tracefs_kprobe_clear_all() to remove all kprobes
  libtracefs: Implement tracefs_kprobe_clear_probe()
  libtracefs: Add man pages for kprobe functions

 Documentation/libtracefs-kprobes.txt | 275 +++++++++++++++++++++
 include/tracefs.h                    |   9 +
 src/Makefile                         |   1 +
 src/tracefs-instance.c               |  78 ++++++
 src/tracefs-kprobes.c                | 349 +++++++++++++++++++++++++++
 5 files changed, 712 insertions(+)
 create mode 100644 Documentation/libtracefs-kprobes.txt
 create mode 100644 src/tracefs-kprobes.c

-- 
2.30.2

[PATCH v4 1/7] libtracefs: Implement tracefs_instances()

[Steven Rostedt]

From: "Steven Rostedt (VMware)" <rostedt@goodmis.org>

Implement tracefs_instances() that will take a regex (or NULL for all) and
return a list of instances in the system.

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
---
 include/tracefs.h      |  1 +
 src/tracefs-instance.c | 78 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)

diff --git a/include/tracefs.h b/include/tracefs.h
index da8ad4189d4d..a21d2d2f22a6 100644
--- a/include/tracefs.h
+++ b/include/tracefs.h
@@ -44,6 +44,7 @@ int tracefs_instance_file_read_number(struct tracefs_instance *instance,
 int tracefs_instance_file_open(struct tracefs_instance *instance,
 			       const char *file, int mode);
 int tracefs_instances_walk(int (*callback)(const char *, void *), void *context);
+char **tracefs_instances(const char *regex);
 
 bool tracefs_instance_exists(const char *name);
 bool tracefs_file_exists(struct tracefs_instance *instance, const char *name);
diff --git a/src/tracefs-instance.c b/src/tracefs-instance.c
index 2aeb529903bd..d833fae0fb0c 100644
--- a/src/tracefs-instance.c
+++ b/src/tracefs-instance.c
@@ -14,6 +14,7 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <dirent.h>
+#include <regex.h>
 #include <limits.h>
 #include <pthread.h>
 #include "tracefs.h"
@@ -630,6 +631,83 @@ out:
 	return fret;
 }
 
+static inline bool match(const char *str, regex_t *re)
+{
+	if (!re)
+		return true;
+	return regexec(re, str, 0, NULL, 0) == 0;
+}
+
+struct instance_list {
+	regex_t		*re;
+	char		**list;
+	int		size;
+	int		failed;
+};
+
+static int build_list(const char *name, void *data)
+{
+	struct instance_list *list = data;
+	char **instances;
+	int ret = -1;
+
+	if (!match(name, list->re))
+		return 0;
+
+	instances = realloc(list->list, list->size + 2);
+	if (!instances)
+		goto out;
+
+	list->list = instances;
+	list->list[list->size] = strdup(name);
+	if (!list->list[list->size])
+		goto out;
+
+	list->size++;
+	ret = 0;
+
+ out:
+	list->failed = ret;
+	return ret;
+}
+
+/**
+ * tracefs_instances - return a list of instance names
+ * @regex: A regex of instances to filter on (NULL to match all)
+ *
+ * Returns a list of names of existing instances, that must be
+ * freed with tracefs_list_free(). Note, if there are no matches
+ * then an empty list will be returned (not NULL).
+ * NULL on error.
+ */
+char **tracefs_instances(const char *regex)
+{
+	struct instance_list list = { .re = NULL, .list = NULL };
+	regex_t re;
+	int ret;
+
+	if (regex) {
+		ret = regcomp(&re, regex, REG_ICASE|REG_NOSUB);
+		if (ret < 0)
+			return NULL;
+		list.re = &re;
+	}
+
+	ret = tracefs_instances_walk(build_list, &list);
+	if (ret < 0 || list.failed) {
+		tracefs_list_free(list.list);
+		list.list = NULL;
+	} else {
+		if (!list.list) {
+			/* No matches should produce an empty list */
+			list.list = malloc(sizeof(*list.list));
+			if (list.list)
+				list.list[0] = NULL;
+		}
+	}
+	return list.list;
+}
+
 /**
  * tracefs_get_clock - Get the current trace clock
  * @instance: ftrace instance, can be NULL for the top instance
-- 
2.30.2

Re: [PATCH v4 1/7] libtracefs: Implement tracefs_instances()

[Yordan Karadzhov (VMware)]

On 2.07.21 г. 6:54, Steven Rostedt wrote:
> From: "Steven Rostedt (VMware)" <rostedt@goodmis.org>
> 
> Implement tracefs_instances() that will take a regex (or NULL for all) and
> return a list of instances in the system.
> 
> Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
> ---
>   include/tracefs.h      |  1 +
>   src/tracefs-instance.c | 78 ++++++++++++++++++++++++++++++++++++++++++
>   2 files changed, 79 insertions(+)
> 
> diff --git a/include/tracefs.h b/include/tracefs.h
> index da8ad4189d4d..a21d2d2f22a6 100644
> --- a/include/tracefs.h
> +++ b/include/tracefs.h
> @@ -44,6 +44,7 @@ int tracefs_instance_file_read_number(struct tracefs_instance *instance,
>   int tracefs_instance_file_open(struct tracefs_instance *instance,
>   			       const char *file, int mode);
>   int tracefs_instances_walk(int (*callback)(const char *, void *), void *context);
> +char **tracefs_instances(const char *regex);
>   
>   bool tracefs_instance_exists(const char *name);
>   bool tracefs_file_exists(struct tracefs_instance *instance, const char *name);
> diff --git a/src/tracefs-instance.c b/src/tracefs-instance.c
> index 2aeb529903bd..d833fae0fb0c 100644
> --- a/src/tracefs-instance.c
> +++ b/src/tracefs-instance.c
> @@ -14,6 +14,7 @@
>   #include <sys/stat.h>
>   #include <fcntl.h>
>   #include <dirent.h>
> +#include <regex.h>
>   #include <limits.h>
>   #include <pthread.h>
>   #include "tracefs.h"
> @@ -630,6 +631,83 @@ out:
>   	return fret;
>   }
>   
> +static inline bool match(const char *str, regex_t *re)
> +{
> +	if (!re)
> +		return true;
> +	return regexec(re, str, 0, NULL, 0) == 0;
> +}
> +
> +struct instance_list {
> +	regex_t		*re;
> +	char		**list;
> +	int		size;
> +	int		failed;
> +};
> +
> +static int build_list(const char *name, void *data)
> +{
> +	struct instance_list *list = data;
> +	char **instances;
> +	int ret = -1;
> +
> +	if (!match(name, list->re))
> +		return 0;
> +
> +	instances = realloc(list->list, list->size + 2);
> +	if (!instances)
> +		goto out;
> +
> +	list->list = instances;
Hi Steven,

I am not sure what was your original intention here, but this doesn't 
seem to work properly. Maybe we need something like this:

	int size =  sizeof(*instances) * (list->size + 2);
	instances = realloc(list->list, size);
	if (!instances)
		goto out;

	instances[list->size + 1] = NULL;




> +	list->list[list->size] = strdup(name);
> +	if (!list->list[list->size])
> +		goto out;
> +
> +	list->size++;
> +	ret = 0;
> +
> + out:
> +	list->failed = ret;
> +	return ret;
> +}
> +
> +/**
> + * tracefs_instances - return a list of instance names
> + * @regex: A regex of instances to filter on (NULL to match all)
> + *
> + * Returns a list of names of existing instances, that must be
> + * freed with tracefs_list_free(). Note, if there are no matches
> + * then an empty list will be returned (not NULL).
> + * NULL on error.
> + */
> +char **tracefs_instances(const char *regex)
> +{
> +	struct instance_list list = { .re = NULL, .list = NULL };
> +	regex_t re;
> +	int ret;
> +
> +	if (regex) {
> +		ret = regcomp(&re, regex, REG_ICASE|REG_NOSUB);
> +		if (ret < 0)
> +			return NULL;
> +		list.re = &re;
> +	}
> +
> +	ret = tracefs_instances_walk(build_list, &list);
> +	if (ret < 0 || list.failed) {
> +		tracefs_list_free(list.list);
> +		list.list = NULL;
> +	} else {
> +		if (!list.list) {
> +			/* No matches should produce an empty list */
> +			list.list = malloc(sizeof(*list.list));
> +			if (list.list)
> +				list.list[0] = NULL;
> +		}

Or you can just do:
				return calloc(1, sizeof(*list.list));

Thanks!
Yordan

> +	}
> +	return list.list;
> +}
> +
>   /**
>    * tracefs_get_clock - Get the current trace clock
>    * @instance: ftrace instance, can be NULL for the top instance
> 

Re: [PATCH v4 1/7] libtracefs: Implement tracefs_instances()

[Steven Rostedt]

On Fri, 2 Jul 2021 14:10:46 +0300
"Yordan Karadzhov (VMware)" <y.karadz@gmail.com> wrote:

> > +static int build_list(const char *name, void *data)
> > +{
> > +	struct instance_list *list = data;
> > +	char **instances;
> > +	int ret = -1;
> > +
> > +	if (!match(name, list->re))
> > +		return 0;
> > +
> > +	instances = realloc(list->list, list->size + 2);
> > +	if (!instances)
> > +		goto out;
> > +
> > +	list->list = instances;  
> Hi Steven,
> 
> I am not sure what was your original intention here, but this doesn't 
> seem to work properly. Maybe we need something like this:
> 
> 	int size =  sizeof(*instances) * (list->size + 2);
> 	instances = realloc(list->list, size);

Bah, that's what I get for trying to "multitask" while coding :-p
That was suppose to be:

	instances = realloc(list->list, sizeof(*instances) * (list->size + 2);

as you stated, but I'll keep it one line.

> 	if (!instances)
> 		goto out;
> 
> 	instances[list->size + 1] = NULL;
> 
> 
> 
> 
> > +	list->list[list->size] = strdup(name);
> > +	if (!list->list[list->size])
> > +		goto out;
> > +
> > +	list->size++;
> > +	ret = 0;
> > +
> > + out:
> > +	list->failed = ret;
> > +	return ret;
> > +}
> > +
> > +/**
> > + * tracefs_instances - return a list of instance names
> > + * @regex: A regex of instances to filter on (NULL to match all)
> > + *
> > + * Returns a list of names of existing instances, that must be
> > + * freed with tracefs_list_free(). Note, if there are no matches
> > + * then an empty list will be returned (not NULL).
> > + * NULL on error.
> > + */
> > +char **tracefs_instances(const char *regex)
> > +{
> > +	struct instance_list list = { .re = NULL, .list = NULL };
> > +	regex_t re;
> > +	int ret;
> > +
> > +	if (regex) {
> > +		ret = regcomp(&re, regex, REG_ICASE|REG_NOSUB);
> > +		if (ret < 0)
> > +			return NULL;
> > +		list.re = &re;
> > +	}
> > +
> > +	ret = tracefs_instances_walk(build_list, &list);
> > +	if (ret < 0 || list.failed) {
> > +		tracefs_list_free(list.list);
> > +		list.list = NULL;
> > +	} else {
> > +		if (!list.list) {
> > +			/* No matches should produce an empty list */
> > +			list.list = malloc(sizeof(*list.list));
> > +			if (list.list)
> > +				list.list[0] = NULL;
> > +		}  
> 
> Or you can just do:
> 				return calloc(1, sizeof(*list.list));

Which I do in tracefs_get_probes(), must have missed this one.

Will update, thanks for the review.


-- Steve


> 
> Thanks!
> Yordan
> 
> > +	}
> > +	return list.list;
> > +}

Re: [PATCH v4 1/7] libtracefs: Implement tracefs_instances()

[Yordan Karadzhov (VMware)]

On 2.07.21 г. 14:33, Steven Rostedt wrote:
> On Fri, 2 Jul 2021 14:10:46 +0300
> "Yordan Karadzhov (VMware)" <y.karadz@gmail.com> wrote:
> 
>>> +static int build_list(const char *name, void *data)
>>> +{
>>> +	struct instance_list *list = data;
>>> +	char **instances;
>>> +	int ret = -1;
>>> +
>>> +	if (!match(name, list->re))
>>> +		return 0;
>>> +
>>> +	instances = realloc(list->list, list->size + 2);
>>> +	if (!instances)
>>> +		goto out;
>>> +
>>> +	list->list = instances;
>> Hi Steven,
>>
>> I am not sure what was your original intention here, but this doesn't
>> seem to work properly. Maybe we need something like this:
>>
>> 	int size =  sizeof(*instances) * (list->size + 2);
>> 	instances = realloc(list->list, size);
> 
> Bah, that's what I get for trying to "multitask" while coding :-p
> That was suppose to be:
> 
> 	instances = realloc(list->list, sizeof(*instances) * (list->size + 2);
> 
> as you stated, but I'll keep it one line.
> 
>> 	if (!instances)
>> 		goto out;
>>
>> 	instances[list->size + 1] = NULL;

but you still need to make sure it is NULL terminated.
Y.

>>
>>
>>
>>
>>> +	list->list[list->size] = strdup(name);
>>> +	if (!list->list[list->size])
>>> +		goto out;
>>> +
>>> +	list->size++;
>>> +	ret = 0;
>>> +
>>> + out:
>>> +	list->failed = ret;
>>> +	return ret;
>>> +}
>>> +
>>> +/**
>>> + * tracefs_instances - return a list of instance names
>>> + * @regex: A regex of instances to filter on (NULL to match all)
>>> + *
>>> + * Returns a list of names of existing instances, that must be
>>> + * freed with tracefs_list_free(). Note, if there are no matches
>>> + * then an empty list will be returned (not NULL).
>>> + * NULL on error.
>>> + */
>>> +char **tracefs_instances(const char *regex)
>>> +{
>>> +	struct instance_list list = { .re = NULL, .list = NULL };
>>> +	regex_t re;
>>> +	int ret;
>>> +
>>> +	if (regex) {
>>> +		ret = regcomp(&re, regex, REG_ICASE|REG_NOSUB);
>>> +		if (ret < 0)
>>> +			return NULL;
>>> +		list.re = &re;
>>> +	}
>>> +
>>> +	ret = tracefs_instances_walk(build_list, &list);
>>> +	if (ret < 0 || list.failed) {
>>> +		tracefs_list_free(list.list);
>>> +		list.list = NULL;
>>> +	} else {
>>> +		if (!list.list) {
>>> +			/* No matches should produce an empty list */
>>> +			list.list = malloc(sizeof(*list.list));
>>> +			if (list.list)
>>> +				list.list[0] = NULL;
>>> +		}
>>
>> Or you can just do:
>> 				return calloc(1, sizeof(*list.list));
> 
> Which I do in tracefs_get_probes(), must have missed this one.
> 
> Will update, thanks for the review.
> 
> 
> -- Steve
> 
> 
>>
>> Thanks!
>> Yordan
>>
>>> +	}
>>> +	return list.list;
>>> +}

Re: [PATCH v4 1/7] libtracefs: Implement tracefs_instances()

[Steven Rostedt]

On Fri, 2 Jul 2021 14:39:48 +0300
"Yordan Karadzhov (VMware)" <y.karadz@gmail.com> wrote:

> > as you stated, but I'll keep it one line.
> >   
> >> 	if (!instances)
> >> 		goto out;
> >>
> >> 	instances[list->size + 1] = NULL;  
> 
> but you still need to make sure it is NULL terminated.

Ah I missed that part of your comment (I'm still sleeping), will send
a v2 ;-)

-- Steve

[PATCH v4 2/7] libtracefs: Implement tracefs_kprobe_raw()

[Steven Rostedt]

From: "Steven Rostedt (VMware)" <rostedt@goodmis.org>

Add a function to facilitate creating a kprobe event. It has the "raw" in
its name because it still requires knowing the format of the kprobe. But
does handle the kprobe naming better and writing to the kprobe_event file.

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
---
 include/tracefs.h     |  4 +++
 src/Makefile          |  1 +
 src/tracefs-kprobes.c | 72 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 77 insertions(+)
 create mode 100644 src/tracefs-kprobes.c

diff --git a/include/tracefs.h b/include/tracefs.h
index a21d2d2f22a6..bc504bcb0188 100644
--- a/include/tracefs.h
+++ b/include/tracefs.h
@@ -214,4 +214,8 @@ int tracefs_tracer_clear(struct tracefs_instance *instance);
 ssize_t tracefs_trace_pipe_stream(int fd, struct tracefs_instance *instance, int flags);
 ssize_t tracefs_trace_pipe_print(struct tracefs_instance *instance, int flags);
 void tracefs_trace_pipe_stop(struct tracefs_instance *instance);
+
+int tracefs_kprobe_raw(const char *system, const char *event,
+		       const char *addr, const char *format);
+
 #endif /* _TRACE_FS_H */
diff --git a/src/Makefile b/src/Makefile
index b4cff07efc50..0697a047f4bc 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -8,6 +8,7 @@ OBJS += tracefs-instance.o
 OBJS += tracefs-events.o
 OBJS += tracefs-tools.o
 OBJS += tracefs-marker.o
+OBJS += tracefs-kprobes.o
 
 OBJS := $(OBJS:%.o=$(bdir)/%.o)
 DEPS := $(OBJS:$(bdir)/%.o=$(bdir)/.%.d)
diff --git a/src/tracefs-kprobes.c b/src/tracefs-kprobes.c
new file mode 100644
index 000000000000..d50191d55181
--- /dev/null
+++ b/src/tracefs-kprobes.c
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: LGPL-2.1
+/*
+ * Copyright (C) 2021 VMware Inc, Steven Rostedt <rostedt@goodmis.org>
+ *
+ * Updates:
+ * Copyright (C) 2021, VMware, Tzvetomir Stoyanov <tz.stoyanov@gmail.com>
+ *
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+
+#include "tracefs.h"
+#include "tracefs-local.h"
+
+#define KPROBE_EVENTS "kprobe_events"
+
+/**
+ * tracefs_kprobe_raw - Create a kprobe using raw format
+ * @system: The system name (NULL for the default kprobes)
+ * @event: The event to create (NULL to use @addr for the event)
+ * @addr: The function and offset (or address) to insert the probe
+ * @format: The raw format string to define the probe.
+ *
+ * Create a kprobe that will be in the @system group (or kprobes if
+ * @system is NULL). Have the name of @event (or @addr if @event is
+ * NULL). Will be inserted to @addr (function name, with or without
+ * offset, or a address). And the @format will define the raw format
+ * of the kprobe. See the Linux documentation file under:
+ * Documentation/trace/kprobetrace.rst
+ *
+ * Return 0 on success, or -1 on error.
+ *   If the syntex of @format was incorrect, running
+ *   tracefs_error_last(NULL) may show what went wrong.
+ *
+ * errno will be set to EBADMSG if addr or format is NULL.
+ */
+int tracefs_kprobe_raw(const char *system, const char *event,
+		       const char *addr, const char *format)
+{
+	char *str;
+	int ret;
+
+	if (!tracefs_file_exists(NULL, KPROBE_EVENTS))
+		return -1;
+
+	errno = EBADMSG;
+	if (!addr || !format)
+		return -1;
+
+	if (!event)
+		event = addr;
+
+	if (system)
+		ret = asprintf(&str, "p:%s/%s %s %s\n",
+			       system, event, addr, format);
+	else
+		ret = asprintf(&str, "p:%s %s %s\n",
+			       event, addr, format);
+
+	if (ret < 0)
+		return -1;
+
+	ret = tracefs_instance_file_append(NULL, KPROBE_EVENTS, str);
+	free(str);
+
+	return ret < 0 ? ret : 0;
+}
-- 
2.30.2

[PATCH v4 3/7] libtracefs: Implement tracefs_kretprobe_raw()

[Steven Rostedt]

From: "Steven Rostedt (VMware)" <rostedt@goodmis.org>

Add an interface that is similar to tracefs_kprobe_raw() but creates a
"kretprobe". The difference between a kprobe and a kretprobe is that a
kretprobe comes at the end of a function.

See Documentation/trace/kprobetrace.rst in the Linux kernel source code
for more information. Note, kprobes are started with "p" and kretprobe
lines start with an "r".

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
---
 include/tracefs.h     |  3 +-
 src/tracefs-kprobes.c | 85 ++++++++++++++++++++++++++++++-------------
 2 files changed, 61 insertions(+), 27 deletions(-)

diff --git a/include/tracefs.h b/include/tracefs.h
index bc504bcb0188..64164c8c2b20 100644
--- a/include/tracefs.h
+++ b/include/tracefs.h
@@ -217,5 +217,6 @@ void tracefs_trace_pipe_stop(struct tracefs_instance *instance);
 
 int tracefs_kprobe_raw(const char *system, const char *event,
 		       const char *addr, const char *format);
-
+int tracefs_kretprobe_raw(const char *system, const char *event,
+			  const char *addr, const char *format);
 #endif /* _TRACE_FS_H */
diff --git a/src/tracefs-kprobes.c b/src/tracefs-kprobes.c
index d50191d55181..4970620b28f2 100644
--- a/src/tracefs-kprobes.c
+++ b/src/tracefs-kprobes.c
@@ -19,28 +19,9 @@
 
 #define KPROBE_EVENTS "kprobe_events"
 
-/**
- * tracefs_kprobe_raw - Create a kprobe using raw format
- * @system: The system name (NULL for the default kprobes)
- * @event: The event to create (NULL to use @addr for the event)
- * @addr: The function and offset (or address) to insert the probe
- * @format: The raw format string to define the probe.
- *
- * Create a kprobe that will be in the @system group (or kprobes if
- * @system is NULL). Have the name of @event (or @addr if @event is
- * NULL). Will be inserted to @addr (function name, with or without
- * offset, or a address). And the @format will define the raw format
- * of the kprobe. See the Linux documentation file under:
- * Documentation/trace/kprobetrace.rst
- *
- * Return 0 on success, or -1 on error.
- *   If the syntex of @format was incorrect, running
- *   tracefs_error_last(NULL) may show what went wrong.
- *
- * errno will be set to EBADMSG if addr or format is NULL.
- */
-int tracefs_kprobe_raw(const char *system, const char *event,
-		       const char *addr, const char *format)
+static int insert_kprobe(const char *type, const char *system,
+			 const char *event, const char *addr,
+			 const char *format)
 {
 	char *str;
 	int ret;
@@ -56,11 +37,11 @@ int tracefs_kprobe_raw(const char *system, const char *event,
 		event = addr;
 
 	if (system)
-		ret = asprintf(&str, "p:%s/%s %s %s\n",
-			       system, event, addr, format);
+		ret = asprintf(&str, "%s:%s/%s %s %s\n",
+			       type, system, event, addr, format);
 	else
-		ret = asprintf(&str, "p:%s %s %s\n",
-			       event, addr, format);
+		ret = asprintf(&str, "%s:%s %s %s\n",
+			       type, event, addr, format);
 
 	if (ret < 0)
 		return -1;
@@ -70,3 +51,55 @@ int tracefs_kprobe_raw(const char *system, const char *event,
 
 	return ret < 0 ? ret : 0;
 }
+
+/**
+ * tracefs_kprobe_raw - Create a kprobe using raw format
+ * @system: The system name (NULL for the default kprobes)
+ * @event: The event to create (NULL to use @addr for the event)
+ * @addr: The function and offset (or address) to insert the probe
+ * @format: The raw format string to define the probe.
+ *
+ * Create a kprobe that will be in the @system group (or kprobes if
+ * @system is NULL). Have the name of @event (or @addr if @event is
+ * NULL). Will be inserted to @addr (function name, with or without
+ * offset, or a address). And the @format will define the raw format
+ * of the kprobe. See the Linux documentation file under:
+ * Documentation/trace/kprobetrace.rst
+ *
+ * Return 0 on success, or -1 on error.
+ *   If the syntex of @format was incorrect, running
+ *   tracefs_error_last(NULL) may show what went wrong.
+ *
+ * errno will be set to EBADMSG if addr or format is NULL.
+ */
+int tracefs_kprobe_raw(const char *system, const char *event,
+		       const char *addr, const char *format)
+{
+	return insert_kprobe("p", system, event, addr, format);
+}
+
+/**
+ * tracefs_kretprobe_raw - Create a kretprobe using raw format
+ * @system: The system name (NULL for the default kprobes)
+ * @event: The event to create (NULL to use @addr for the event)
+ * @addr: The function and offset (or address) to insert the retprobe
+ * @format: The raw format string to define the retprobe.
+ *
+ * Create a kretprobe that will be in the @system group (or kprobes if
+ * @system is NULL). Have the name of @event (or @addr if @event is
+ * NULL). Will be inserted to @addr (function name, with or without
+ * offset, or a address). And the @format will define the raw format
+ * of the kprobe. See the Linux documentation file under:
+ * Documentation/trace/kprobetrace.rst
+ *
+ * Return 0 on success, or -1 on error.
+ *   If the syntex of @format was incorrect, running
+ *   tracefs_error_last(NULL) may show what went wrong.
+ *
+ * errno will be set to EBADMSG if addr or format is NULL.
+ */
+int tracefs_kretprobe_raw(const char *system, const char *event,
+			  const char *addr, const char *format)
+{
+	return insert_kprobe("r", system, event, addr, format);
+}
-- 
2.30.2

[PATCH v4 4/7] libtracefs: Implement tracefs_get_kprobes()

[Steven Rostedt]

From: "Steven Rostedt (VMware)" <rostedt@goodmis.org>

Add the function tracefs_get_kprobes() that returns a list of kprobes that
are registered. The list contains strings that are of the format
"group/event" (i.e. "kprobes/open"). The last element in the list is a
NULL pointer. In the case that there are no kprobes, the list will return
a single entry of a NULL pointer.

NULL is returned in the case of error (mainly a memory issue, or bad
parsing).

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
---
 include/tracefs.h     |  1 +
 src/tracefs-kprobes.c | 66 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+)

diff --git a/include/tracefs.h b/include/tracefs.h
index 64164c8c2b20..5f59c480d572 100644
--- a/include/tracefs.h
+++ b/include/tracefs.h
@@ -219,4 +219,5 @@ int tracefs_kprobe_raw(const char *system, const char *event,
 		       const char *addr, const char *format);
 int tracefs_kretprobe_raw(const char *system, const char *event,
 			  const char *addr, const char *format);
+char **tracefs_get_kprobes(void);
 #endif /* _TRACE_FS_H */
diff --git a/src/tracefs-kprobes.c b/src/tracefs-kprobes.c
index 4970620b28f2..e875d6e8a65f 100644
--- a/src/tracefs-kprobes.c
+++ b/src/tracefs-kprobes.c
@@ -103,3 +103,69 @@ int tracefs_kretprobe_raw(const char *system, const char *event,
 {
 	return insert_kprobe("r", system, event, addr, format);
 }
+
+/**
+ * tracefs_get_kprobes - return a list kprobes (by group/event name)
+ *
+ * Returns a list of strings that contain the kprobes that exist
+ * in the kprobe_events files. The strings returned are in the
+ * "group/event" format.
+ * The list must be freed with tracefs_list_free().
+ * If there are no kprobes, a list is still returned, but it contains
+ * only a NULL pointer.
+ * On error, NULL is returned.
+ */
+char **tracefs_get_kprobes(void)
+{
+	char **list = NULL;
+	char *content;
+	char *saveptr;
+	char *event;
+	char *p;
+	int cnt = 0;
+
+	content = tracefs_instance_file_read(NULL, KPROBE_EVENTS, NULL);
+	if (!content)
+		return NULL;
+
+	p = strtok_r(content, ":", &saveptr);
+
+	while (p) {
+		char **tmp;
+
+		/* Failed parsing always return a failure */
+		p = strtok_r(NULL, " ", &saveptr);
+		if (!p)
+			break;
+
+		event = strdup(p);
+		if (!event)
+			goto fail;
+
+		tmp = realloc(list, sizeof(*list) * (cnt + 2));
+		if (!tmp)
+			goto fail;
+
+		list = tmp;
+		list[cnt++] = event;
+		list[cnt] = NULL;
+
+		p = strtok_r(NULL, "\n", &saveptr);
+		/* Could be end of content */
+		if (!p)
+			break;
+
+		/* p is NULL on end of content */
+		p = strtok_r(NULL, ":", &saveptr);
+	}
+
+	if (!list)
+		list = calloc(1, sizeof(*list));
+ out:
+	free(content);
+	return list;
+ fail:
+	free(list);
+	list = NULL;
+	goto out;
+}
-- 
2.30.2

[PATCH v4 5/7] libtracefs: Implement tracefs_kprobe_clear_all() to remove all kprobes

[Steven Rostedt]

From: "Steven Rostedt (VMware)" <rostedt@goodmis.org>

A call to tracefs_kprobe_clear_all() will attempt to disable all kprobes. If
any kprobe is set, and the @force parameter is set, it will fail with
errno set to EBUSY. If @force is set, then it will attempt to disable all
the defined kprobe events and then clear it.

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
---
 include/tracefs.h     |   1 +
 src/tracefs-kprobes.c | 103 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+)

diff --git a/include/tracefs.h b/include/tracefs.h
index 5f59c480d572..3b57c596feab 100644
--- a/include/tracefs.h
+++ b/include/tracefs.h
@@ -220,4 +220,5 @@ int tracefs_kprobe_raw(const char *system, const char *event,
 int tracefs_kretprobe_raw(const char *system, const char *event,
 			  const char *addr, const char *format);
 char **tracefs_get_kprobes(void);
+int tracefs_kprobe_clear_all(bool force);
 #endif /* _TRACE_FS_H */
diff --git a/src/tracefs-kprobes.c b/src/tracefs-kprobes.c
index e875d6e8a65f..fe61a25baa10 100644
--- a/src/tracefs-kprobes.c
+++ b/src/tracefs-kprobes.c
@@ -169,3 +169,106 @@ char **tracefs_get_kprobes(void)
 	list = NULL;
 	goto out;
 }
+
+struct instance_list {
+	struct instance_list	*next;
+	struct tracefs_instance	*instance;
+};
+
+static void disable_events(const char *system, const char *event,
+			   char **list)
+{
+	struct tracefs_instance *instance;
+	int i;
+
+	/*
+	 * Note, this will not fail even on error.
+	 * That is because even if something fails, it may still
+	 * work enough to clear the kprobes. If that's the case
+	 * the clearing after the loop will succeed and the function
+	 * is a success, even though other parts had failed. If
+	 * one of the kprobe events is enabled in one of the
+	 * instances that fail, then the clearing will fail too
+	 * and the function will return an error.
+	 */
+
+	tracefs_event_disable(NULL, system, event);
+	/* No need to test results */
+
+	if (!list)
+		return;
+
+	for (i = 0; list[i]; i++) {
+		instance = tracefs_instance_alloc(NULL, list[i]);
+		/* If this fails, try the next one */
+		if (!instance)
+			continue;
+		tracefs_event_disable(instance, system, event);
+		tracefs_instance_free(instance);
+	}
+	return;
+}
+
+/**
+ * tracefs_kprobe_clear_all - clear kprobe events
+ * @force: Will attempt to disable all kprobe events and clear them
+ *
+ * Will remove all defined kprobe events. If any of them are enabled,
+ * and @force is not set, then it will error with -1 and errno to be
+ * EBUSY. If @force is set, then it will attempt to disable all the kprobe
+ * events in all instances, and try again.
+ *
+ * Returns zero on success, -1 otherwise.
+ */
+int tracefs_kprobe_clear_all(bool force)
+{
+	char **instance_list;
+	char **kprobe_list;
+	char *saveptr;
+	char *system;
+	char *kprobe;
+	char *event;
+	int ret;
+	int i;
+
+	ret = tracefs_instance_file_clear(NULL, KPROBE_EVENTS);
+	if (!ret)
+		return 0;
+
+	if (!force)
+		return -1;
+
+	kprobe_list = tracefs_get_kprobes();
+	if (!kprobe_list)
+		return -1;
+
+	instance_list = tracefs_instances(NULL);
+	/*
+	 * Even if the above failed and instance_list is NULL,
+	 * keep going, as the enabled event may simply be in the
+	 * top level.
+	 */
+
+	for (i = 0; kprobe_list[i]; i++) {
+		kprobe = kprobe_list[i];
+
+		system = strtok_r(kprobe, "/", &saveptr);
+		if (!system)
+			goto out;
+
+		event = strtok_r(NULL," ", &saveptr);
+		if (!event)
+			goto out;
+
+		disable_events(system, event, instance_list);
+
+		ret = tracefs_instance_file_clear(NULL, KPROBE_EVENTS);
+		/* On success stop the loop */
+		if (!ret)
+			goto out;
+	}
+ out:
+	tracefs_list_free(instance_list);
+	tracefs_list_free(kprobe_list);
+	return ret;
+}
-- 
2.30.2

Re: [PATCH v4 5/7] libtracefs: Implement tracefs_kprobe_clear_all() to remove all kprobes

[Yordan Karadzhov (VMware)]

On 2.07.21 г. 6:54, Steven Rostedt wrote:
> From: "Steven Rostedt (VMware)" <rostedt@goodmis.org>
> 
> A call to tracefs_kprobe_clear_all() will attempt to disable all kprobes. If
> any kprobe is set, and the @force parameter is set, it will fail with
> errno set to EBUSY. If @force is set, then it will attempt to disable all
> the defined kprobe events and then clear it.
> 
> Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
> ---
>   include/tracefs.h     |   1 +
>   src/tracefs-kprobes.c | 103 ++++++++++++++++++++++++++++++++++++++++++
>   2 files changed, 104 insertions(+)
> 
> diff --git a/include/tracefs.h b/include/tracefs.h
> index 5f59c480d572..3b57c596feab 100644
> --- a/include/tracefs.h
> +++ b/include/tracefs.h
> @@ -220,4 +220,5 @@ int tracefs_kprobe_raw(const char *system, const char *event,
>   int tracefs_kretprobe_raw(const char *system, const char *event,
>   			  const char *addr, const char *format);
>   char **tracefs_get_kprobes(void);
> +int tracefs_kprobe_clear_all(bool force);
>   #endif /* _TRACE_FS_H */
> diff --git a/src/tracefs-kprobes.c b/src/tracefs-kprobes.c
> index e875d6e8a65f..fe61a25baa10 100644
> --- a/src/tracefs-kprobes.c
> +++ b/src/tracefs-kprobes.c
> @@ -169,3 +169,106 @@ char **tracefs_get_kprobes(void)
>   	list = NULL;
>   	goto out;
>   }
> +
> +struct instance_list {
> +	struct instance_list	*next;
> +	struct tracefs_instance	*instance;
> +};

This is not used in this source file. I see that it was moved to 
tracefs-instance.c

Thanks!
Y.

> +
> +static void disable_events(const char *system, const char *event,
> +			   char **list)
> +{
> +	struct tracefs_instance *instance;
> +	int i;
> +
> +	/*
> +	 * Note, this will not fail even on error.
> +	 * That is because even if something fails, it may still
> +	 * work enough to clear the kprobes. If that's the case
> +	 * the clearing after the loop will succeed and the function
> +	 * is a success, even though other parts had failed. If
> +	 * one of the kprobe events is enabled in one of the
> +	 * instances that fail, then the clearing will fail too
> +	 * and the function will return an error.
> +	 */
> +
> +	tracefs_event_disable(NULL, system, event);
> +	/* No need to test results */
> +
> +	if (!list)
> +		return;
> +
> +	for (i = 0; list[i]; i++) {
> +		instance = tracefs_instance_alloc(NULL, list[i]);
> +		/* If this fails, try the next one */
> +		if (!instance)
> +			continue;
> +		tracefs_event_disable(instance, system, event);
> +		tracefs_instance_free(instance);
> +	}
> +	return;
> +}
> +
> +/**
> + * tracefs_kprobe_clear_all - clear kprobe events
> + * @force: Will attempt to disable all kprobe events and clear them
> + *
> + * Will remove all defined kprobe events. If any of them are enabled,
> + * and @force is not set, then it will error with -1 and errno to be
> + * EBUSY. If @force is set, then it will attempt to disable all the kprobe
> + * events in all instances, and try again.
> + *
> + * Returns zero on success, -1 otherwise.
> + */
> +int tracefs_kprobe_clear_all(bool force)
> +{
> +	char **instance_list;
> +	char **kprobe_list;
> +	char *saveptr;
> +	char *system;
> +	char *kprobe;
> +	char *event;
> +	int ret;
> +	int i;
> +
> +	ret = tracefs_instance_file_clear(NULL, KPROBE_EVENTS);
> +	if (!ret)
> +		return 0;
> +
> +	if (!force)
> +		return -1;
> +
> +	kprobe_list = tracefs_get_kprobes();
> +	if (!kprobe_list)
> +		return -1;
> +
> +	instance_list = tracefs_instances(NULL);
> +	/*
> +	 * Even if the above failed and instance_list is NULL,
> +	 * keep going, as the enabled event may simply be in the
> +	 * top level.
> +	 */
> +
> +	for (i = 0; kprobe_list[i]; i++) {
> +		kprobe = kprobe_list[i];
> +
> +		system = strtok_r(kprobe, "/", &saveptr);
> +		if (!system)
> +			goto out;
> +
> +		event = strtok_r(NULL," ", &saveptr);
> +		if (!event)
> +			goto out;
> +
> +		disable_events(system, event, instance_list);
> +
> +		ret = tracefs_instance_file_clear(NULL, KPROBE_EVENTS);
> +		/* On success stop the loop */
> +		if (!ret)
> +			goto out;
> +	}
> + out:
> +	tracefs_list_free(instance_list);
> +	tracefs_list_free(kprobe_list);
> +	return ret;
> +}
> 

Re: [PATCH v4 5/7] libtracefs: Implement tracefs_kprobe_clear_all() to remove all kprobes

[Steven Rostedt]

On Fri, 2 Jul 2021 14:15:44 +0300
"Yordan Karadzhov (VMware)" <y.karadz@gmail.com> wrote:

> > diff --git a/src/tracefs-kprobes.c b/src/tracefs-kprobes.c
> > index e875d6e8a65f..fe61a25baa10 100644
> > --- a/src/tracefs-kprobes.c
> > +++ b/src/tracefs-kprobes.c
> > @@ -169,3 +169,106 @@ char **tracefs_get_kprobes(void)
> >   	list = NULL;
> >   	goto out;
> >   }
> > +
> > +struct instance_list {
> > +	struct instance_list	*next;
> > +	struct tracefs_instance	*instance;
> > +};  
> 
> This is not used in this source file. I see that it was moved to 
> tracefs-instance.c

Internally, this was my fourth iteration between v2 and v3, and I kept
finding "better" ways to write this code. This was left over from one
of those iterations.

I'll nuke it. Thanks!

-- Steve

[PATCH v4 6/7] libtracefs: Implement tracefs_kprobe_clear_probe()

[Steven Rostedt]

From: "Steven Rostedt (VMware)" <rostedt@goodmis.org>

Add the function tracefs_kprobe_clear_probe() that will remove a single
kprobe. If the @force parameter is set, it will disable that probe in all
instances (including the top level instance) before removing it.

If the @event parameter is NULL, then it will clear all events that are
defined by the @system parameter. If the @system parameter is NULL,
then it will use the default "kprobes" group.

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
---
 include/tracefs.h     |   1 +
 src/tracefs-kprobes.c | 123 +++++++++++++++++++++++++++++++++---------
 2 files changed, 100 insertions(+), 24 deletions(-)

diff --git a/include/tracefs.h b/include/tracefs.h
index 3b57c596feab..2771fad6d0ef 100644
--- a/include/tracefs.h
+++ b/include/tracefs.h
@@ -221,4 +221,5 @@ int tracefs_kretprobe_raw(const char *system, const char *event,
 			  const char *addr, const char *format);
 char **tracefs_get_kprobes(void);
 int tracefs_kprobe_clear_all(bool force);
+int tracefs_kprobe_clear_probe(const char *system, const char *event, bool force);
 #endif /* _TRACE_FS_H */
diff --git a/src/tracefs-kprobes.c b/src/tracefs-kprobes.c
index fe61a25baa10..7ca34fe6c7b7 100644
--- a/src/tracefs-kprobes.c
+++ b/src/tracefs-kprobes.c
@@ -209,18 +209,17 @@ static void disable_events(const char *system, const char *event,
 	return;
 }
 
-/**
- * tracefs_kprobe_clear_all - clear kprobe events
- * @force: Will attempt to disable all kprobe events and clear them
- *
- * Will remove all defined kprobe events. If any of them are enabled,
- * and @force is not set, then it will error with -1 and errno to be
- * EBUSY. If @force is set, then it will attempt to disable all the kprobe
- * events in all instances, and try again.
- *
- * Returns zero on success, -1 otherwise.
- */
-int tracefs_kprobe_clear_all(bool force)
+static int clear_kprobe(const char *system, const char *event)
+{
+	/* '-' + ':' + '/' + '\n' + '\0' = 5 bytes */
+	int len = strlen(system) + strlen(event) + 5;
+	char content[len];
+
+	sprintf(content, "-:%s/%s", system, event);
+	return tracefs_instance_file_append(NULL, KPROBE_EVENTS, content);
+}
+
+static int kprobe_clear_probes(const char *group, bool force)
 {
 	char **instance_list;
 	char **kprobe_list;
@@ -231,13 +230,6 @@ int tracefs_kprobe_clear_all(bool force)
 	int ret;
 	int i;
 
-	ret = tracefs_instance_file_clear(NULL, KPROBE_EVENTS);
-	if (!ret)
-		return 0;
-
-	if (!force)
-		return -1;
-
 	kprobe_list = tracefs_get_kprobes();
 	if (!kprobe_list)
 		return -1;
@@ -249,6 +241,13 @@ int tracefs_kprobe_clear_all(bool force)
 	 * top level.
 	 */
 
+	/*
+	 * If a system is defined, the default is to pass unless
+	 * an event fails to be removed. If a system is not defined,
+	 * the default is to fail, unless all are removed.
+	 */
+	ret = group ? 0 : -1;
+
 	for (i = 0; kprobe_list[i]; i++) {
 		kprobe = kprobe_list[i];
 
@@ -260,15 +259,91 @@ int tracefs_kprobe_clear_all(bool force)
 		if (!event)
 			goto out;
 
-		disable_events(system, event, instance_list);
+		/* Skip if this does not match a given system */
+		if (group && strcmp(system, group) != 0)
+			continue;
 
-		ret = tracefs_instance_file_clear(NULL, KPROBE_EVENTS);
-		/* On success stop the loop */
-		if (!ret)
-			goto out;
+		if (force)
+			disable_events(system, event, instance_list);
+
+		if (group) {
+			ret = clear_kprobe(system, event);
+			if (ret < 0)
+				goto out;
+		} else {
+			ret = tracefs_instance_file_clear(NULL, KPROBE_EVENTS);
+			/* On success stop the loop */
+			if (!ret)
+				goto out;
+		}
+
+		/* Set the default for whether a system is defined or not */
+		ret = group ? 0 : -1;
 	}
  out:
 	tracefs_list_free(instance_list);
 	tracefs_list_free(kprobe_list);
 	return ret;
 }
+
+/**
+ * tracefs_kprobe_clear_all - clear kprobe events
+ * @force: Will attempt to disable all kprobe events and clear them
+ *
+ * Will remove all defined kprobe events. If any of them are enabled,
+ * and @force is not set, then it will error with -1 and errno to be
+ * EBUSY. If @force is set, then it will attempt to disable all the kprobe
+ * events in all instances, and try again.
+ *
+ * Returns zero on success, -1 otherwise.
+ */
+int tracefs_kprobe_clear_all(bool force)
+{
+	if (tracefs_instance_file_clear(NULL, KPROBE_EVENTS) == 0)
+		return 0;
+
+	if (!force)
+		return -1;
+
+	/* Attempt to disable all kprobe events */
+	return kprobe_clear_probes(NULL, force);
+}
+
+/**
+ * tracefs_kprobe_clear_all - clear kprobe events
+ * @system: System to clear (NULL means default)
+ * @event: Name of probe to clear in system (NULL for all probes in system)
+ * @force: Will attempt to disable all kprobe events and clear them
+ *
+ * Will remove the kprobes that match the @system and @event. If @system
+ * is NULL, then "kprobes" is used and will ignore all other system
+ * groups of kprobes. The @event is NULL then all events under the given
+ * @system are removed, otherwise only the event that matches.
+ *
+ * Returns zero on success, -1 otherwise.
+ */
+int tracefs_kprobe_clear_probe(const char *system, const char *event, bool force)
+{
+	char **instance_list;
+	int ret;
+
+	if (!system)
+		system = "kprobes";
+
+	if (!event)
+		return kprobe_clear_probes(system, force);
+
+	/*
+	 * Since we know we are disabling a specific event, try
+	 * to disable it first before clearing it.
+	 */
+	if (force) {
+		instance_list = tracefs_instances(NULL);
+		disable_events(system, event, instance_list);
+		tracefs_list_free(instance_list);
+	}
+
+	ret = clear_kprobe(system, event);
+
+	return ret < 0 ? -1 : 0;
+}
-- 
2.30.2

[PATCH v4 7/7] libtracefs: Add man pages for kprobe functions

[Steven Rostedt]

From: "Steven Rostedt (VMware)" <rostedt@goodmis.org>

Add man pages that describe the following functions:

    tracefs_kprobe_raw
    tracefs_kretprobe_raw
    tracefs_get_kprobes
    tracefs_kprobe_clear_all
    tracefs_kprobe_clear_probe

Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
---
 Documentation/libtracefs-kprobes.txt | 275 +++++++++++++++++++++++++++
 1 file changed, 275 insertions(+)
 create mode 100644 Documentation/libtracefs-kprobes.txt

diff --git a/Documentation/libtracefs-kprobes.txt b/Documentation/libtracefs-kprobes.txt
new file mode 100644
index 000000000000..0787382ae8b2
--- /dev/null
+++ b/Documentation/libtracefs-kprobes.txt
@@ -0,0 +1,275 @@
+libtracefs(3)
+=============
+
+NAME
+----
+tracefs_kprobe_raw, tracefs_kretprobe_raw, tracefs_get_kprobes, tracefs_kprobe_clear_all, tracefs_kprobe_clear_probe - Create, list, and destroy kprobes
+
+SYNOPSIS
+--------
+[verse]
+--
+*#include <tracefs.h>*
+
+int tracefs_kprobe_raw(const char pass:[*]system, const char pass:[*]event, const char pass:[*]addr, const char pass:[*]format);
+int tracefs_kretprobe_raw(const char pass:[*]system, const char pass:[*]event, const char pass:[*]addr, const char pass:[*]format);
+char pass:[*]pass:[*]tracefs_get_kprobes(void);
+int tracefs_kprobe_clear_all(bool force);
+int tracefs_kprobe_clear_probe(const char pass:[*]system, const char pass:[*]event, bool force);
+--
+
+DESCRIPTION
+-----------
+*tracefs_kprobe_raw*() will create a kprobe event. If _system_ is NULL, then
+the default "kprobes" is used for the group (event system). Otherwise if _system_
+is specified then the kprobe will be created under the group by that name. The
+_event_ is the name of the kprobe event to create. The _addr_ can be a function,
+a function and offset, or a kernel address. This is where the location of the
+kprobe will be inserted in the kernel. The _format_ is the kprobe format as
+specified as FETCHARGS in the Linux kernel source in the Documentation/trace/kprobetrace.rst
+document.
+
+*tracefs_kretprobe_raw*() is the same as *tracefs_kprobe_raw()*, except that it
+creates a kretprobe instead of a kprobe. The difference is also described
+in the Linux kernel source in the Documentation/trace/kprobetrace.rst file.
+
+*tracefs_get_kprobes*() returns an array of strings (char pass:[*]) that contain
+the registered kprobes and kretprobes. The names are in the "system/event" format.
+That is, one string holds both the kprobe's name as well as the group it is
+defined under. These strings are allocated and may be modified with the
+*strtok*(3) and *strtok_r*(3) functions. The string returned must be freed with
+*tracefs_list_free*(3).
+
+*tracefs_kprobe_clear_all*() will try to remove all kprobes that have been
+registered. If the @force flag is set, it will then disable those kprobe events
+if they are enabled and then try to clear the kprobes.
+
+*tracefs_kprobe_clear_probe*() will try to clear specified kprobes. If _system_
+is NULL, then it will only clear the default kprobes under the "kprobes" group.
+If _event_ is NULL, it will clear all kprobes under the given _system_. If the
+_force_ flag is set, then it will disable the given kprobe events before clearing
+them.
+
+RETURN VALUE
+------------
+
+*tracefs_kprobe_raw*(), *tracefs_kretprobe_raw*(), *tracefs_kprobe_clear_all*(),
+and *tracefs_kprobe_clear_probe*() return 0 on success, or -1 on error.
+
+If a parsing error occurs on *tracefs_kprobe_raw*() or *tracefs_kretprobe_raw*()
+then *tracefs_error_last*(3) may be used to retrieve the error message explaining
+the parsing issue.
+
+*tracefs_get_kprobes*() returns an allocate string list of allocated strings
+on success that must be freed with *tracefs_list_free*(3) and returns
+NULL on error.
+
+ERRORS
+------
+The following errors are for all the above calls:
+
+*EPERM* Not run as root user
+
+*ENODEV* Kprobe events are not configured for the running kernel.
+
+*ENOMEM* Memory allocation error.
+
+*tracefs_kprobe_raw*(), *tracefs_kretprobe_raw*() can fail with the following errors:
+
+*EBADMSG* Either _addr_ or _format_ are NULL.
+
+*EINVAL*  Most likely a parsing error occurred (use *tracefs_error_last*(3) to possibly
+          see what that error was).
+
+Other errors may also happen caused by internal system calls.
+
+EXAMPLE
+-------
+[source,c]
+--
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/wait.h>
+
+#include <tracefs/tracefs.h>
+
+static struct tep_event *open_event;
+static struct tep_format_field *file_field;
+
+static struct tep_event *openret_event;
+static struct tep_format_field *ret_field;
+
+static int callback(struct tep_event *event, struct tep_record *record,
+		    int cpu, void *data)
+{
+	struct trace_seq seq;
+
+	trace_seq_init(&seq);
+	tep_print_event(event->tep, &seq, record, "%d-%s: ", TEP_PRINT_PID, TEP_PRINT_COMM);
+
+	if (event->id == open_event->id) {
+		trace_seq_puts(&seq, "open file='");
+		tep_print_field(&seq, record->data, file_field);
+		trace_seq_puts(&seq, "'\n");
+	} else if (event->id == openret_event->id) {
+		unsigned long long ret;
+		tep_read_number_field(ret_field, record->data, &ret);
+		trace_seq_printf(&seq, "open ret=%lld\n", ret);
+	} else {
+		goto out;
+	}
+
+	trace_seq_terminate(&seq);
+	trace_seq_do_printf(&seq);
+out:
+	trace_seq_destroy(&seq);
+
+	return 0;
+}
+
+static pid_t run_exec(char **argv, char **env)
+{
+	pid_t pid;
+
+	pid = fork();
+	if (pid)
+		return pid;
+
+	execve(argv[0], argv, env);
+	perror("exec");
+	exit(-1);
+}
+
+const char *mykprobe = "my_kprobes";
+
+enum kprobe_type {
+	KPROBE,
+	KRETPROBE,
+};
+
+static void __kprobe_create(enum kprobe_type type, const char *event,
+			    const char *addr, const char *fmt)
+{
+	char *err;
+	int r;
+
+	if (type == KPROBE)
+		r = tracefs_kprobe_raw(mykprobe, event, addr, fmt);
+	else
+		r = tracefs_kretprobe_raw(mykprobe, event, addr, fmt);
+	if (r < 0) {
+		err = tracefs_error_last(NULL);
+		perror("Failed to create kprobe:");
+		if (err && strlen(err))
+			fprintf(stderr, "%s\n", err);
+	}
+}
+
+static void kprobe_create(const char *event, const char *addr,
+			  const char *fmt)
+{
+	__kprobe_create(KPROBE, event, addr, fmt);
+}
+
+static void kretprobe_create(const char *event, const char *addr,
+			     const char *fmt)
+{
+	__kprobe_create(KRETPROBE, event, addr, fmt);
+}
+
+int main (int argc, char **argv, char **env)
+{
+	struct tracefs_instance *instance;
+	struct tep_handle *tep;
+	const char *sysnames[] = { mykprobe, NULL };
+	pid_t pid;
+
+	if (argc < 2) {
+		printf("usage: %s command\n", argv[0]);
+		exit(-1);
+	}
+
+	instance = tracefs_instance_create("exec_open");
+	if (!instance) {
+		perror("creating instance");
+		exit(-1);
+	}
+
+	tracefs_kprobe_clear_probe(mykprobe, NULL, true);
+
+	kprobe_create("open", "do_sys_openat2",
+		      "file=+0($arg2):ustring flags=+0($arg3):x64 mode=+8($arg3):x64\n");
+
+	kretprobe_create("openret", "do_sys_openat2", "ret=%ax");
+
+	tep = tracefs_local_events_system(NULL, sysnames);
+	if (!tep) {
+		perror("reading events");
+		exit(-1);
+	}
+	open_event = tep_find_event_by_name(tep, mykprobe, "open");
+	file_field = tep_find_field(open_event, "file");
+
+	openret_event = tep_find_event_by_name(tep, mykprobe, "openret");
+	ret_field = tep_find_field(openret_event, "ret");
+
+	tracefs_event_enable(instance, mykprobe, NULL);
+	pid = run_exec(&argv[1], env);
+
+	/* Let the child start to run */
+	sched_yield();
+
+	do {
+		tracefs_load_cmdlines(NULL, tep);
+		tracefs_iterate_raw_events(tep, instance, NULL, 0, callback, NULL);
+	} while (waitpid(pid, NULL, WNOHANG) != pid);
+
+	/* Will disable the events */
+	tracefs_kprobe_clear_probe(mykprobe, NULL, true);
+	tracefs_instance_destroy(instance);
+	tep_free(tep);
+
+	return 0;
+}
+--
+
+FILES
+-----
+[verse]
+--
+*tracefs.h*
+	Header file to include in order to have access to the library APIs.
+*-ltracefs*
+	Linker switch to add when building a program that uses the library.
+--
+
+SEE ALSO
+--------
+_libtracefs(3)_,
+_libtraceevent(3)_,
+_trace-cmd(1)_
+
+AUTHOR
+------
+[verse]
+--
+*Steven Rostedt* <rostedt@goodmis.org>
+*Tzvetomir Stoyanov* <tz.stoyanov@gmail.com>
+*sameeruddin shaik* <sameeruddin.shaik8@gmail.com>
+--
+REPORTING BUGS
+--------------
+Report bugs to  <linux-trace-devel@vger.kernel.org>
+
+LICENSE
+-------
+libtracefs is Free Software licensed under the GNU LGPL 2.1
+
+RESOURCES
+---------
+https://git.kernel.org/pub/scm/libs/libtrace/libtracefs.git/
+
+COPYING
+-------
+Copyright \(C) 2020 VMware, Inc. Free use of this software is granted under
+the terms of the GNU Public License (GPL).
-- 
2.30.2