[PATCH lttng-tools 2/5] Tests: Add a health check utility program

[PATCH lttng-tools 2/5] Tests: Add a health check utility program

[Christian Babeux]

The health_check program is a simple utility to query the health
status of the different threads of the sessiond.

Sample output:

> ./health_check
Health check cmd: 0
Health check app. manage: 0
Health check app. registration: 0
Health check kernel: 0
Health check consumer: 0

The return code is encoded to indicate which thread has failed.

Signed-off-by: Christian Babeux <christian.babeux@efficios.com>
---
 tests/tools/health/Makefile.am    | 20 +++++++++++
 tests/tools/health/health_check.c | 73 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 tests/tools/health/Makefile.am
 create mode 100644 tests/tools/health/health_check.c

diff --git a/tests/tools/health/Makefile.am b/tests/tools/health/Makefile.am
new file mode 100644
index 0000000..09573db
--- /dev/null
+++ b/tests/tools/health/Makefile.am
@@ -0,0 +1,20 @@
+AM_CFLAGS = -I. -O2 -g -I../../../include
+AM_LDFLAGS =
+
+if LTTNG_TOOLS_BUILD_WITH_LIBDL
+AM_LDFLAGS += -ldl
+endif
+if LTTNG_TOOLS_BUILD_WITH_LIBC_DL
+AM_LDFLAGS += -lc
+endif
+
+UTILS=
+
+noinst_PROGRAMS = health_check
+
+health_check_SOURCES = health_check.c $(UTILS)
+health_check_LDADD = $(top_builddir)/src/lib/lttng-ctl/liblttng-ctl.la \
+		     $(top_builddir)/src/common/libcommon.la
+
+noinst_SCRIPTS =
+EXTRA_DIST =
diff --git a/tests/tools/health/health_check.c b/tests/tools/health/health_check.c
new file mode 100644
index 0000000..3eef110
--- /dev/null
+++ b/tests/tools/health/health_check.c
@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2012 - Christian Babeux <christian.babeux@efficios.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License, version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <stdio.h>
+
+#include "lttng/lttng.h"
+
+#define HEALTH_CMD_FAIL     (1 << 0)
+#define HEALTH_APP_MNG_FAIL (1 << 1)
+#define HEALTH_APP_REG_FAIL (1 << 2)
+#define HEALTH_KERNEL_FAIL  (1 << 3)
+#define HEALTH_CSMR_FAIL    (1 << 4)
+
+int main(int argc, char *argv[])
+{
+	int health = -1;
+	int status = 0;
+
+	/* Command thread */
+	health = lttng_health_check(LTTNG_HEALTH_CMD);
+	printf("Health check cmd: %d\n", health);
+
+	if (health) {
+		status |= HEALTH_CMD_FAIL;
+	}
+
+	/* App manage thread */
+	health = lttng_health_check(LTTNG_HEALTH_APP_MANAGE);
+	printf("Health check app. manage: %d\n", health);
+
+	if (health) {
+		status |= HEALTH_APP_MNG_FAIL;
+	}
+	/* App registration thread */
+	health = lttng_health_check(LTTNG_HEALTH_APP_REG);
+	printf("Health check app. registration: %d\n", health);
+
+	if (health) {
+		status |= HEALTH_APP_REG_FAIL;
+	}
+
+	/* Kernel thread */
+	health = lttng_health_check(LTTNG_HEALTH_KERNEL);
+	printf("Health check kernel: %d\n", health);
+
+	if (health) {
+		status |= HEALTH_KERNEL_FAIL;
+	}
+
+	/* Consumer thread */
+	health = lttng_health_check(LTTNG_HEALTH_CONSUMER);
+	printf("Health check consumer: %d\n", health);
+
+	if (health) {
+		status |= HEALTH_CSMR_FAIL;
+	}
+
+	return status;
+}
-- 
1.7.12

[PATCH lttng-tools 3/5] Tests: Add health check thread exit test

[Christian Babeux]

This test trigger a failure in a specified thread using the
recently introduced testpoint mechanism. The testpoints behavior
is implemented in health_exit.c. The testpoint code simply calls
pthread_exit(3) and effectively "kill" the thread without affecting
the other threads behavior.

The test select the thread to be "killed" by enabling a specific
environment variable.

With this test we ensure that each thread can be succesfully terminated
and that the health check feature properly detect a failure.

Signed-off-by: Christian Babeux <christian.babeux@efficios.com>
---
 tests/tools/health/Makefile.am        |   6 ++
 tests/tools/health/health_exit.c      |  80 ++++++++++++++++++++++++++
 tests/tools/health/health_thread_exit | 105 ++++++++++++++++++++++++++++++++++
 3 files changed, 191 insertions(+)
 create mode 100644 tests/tools/health/health_exit.c
 create mode 100755 tests/tools/health/health_thread_exit

diff --git a/tests/tools/health/Makefile.am b/tests/tools/health/Makefile.am
index 09573db..0a3f6c5 100644
--- a/tests/tools/health/Makefile.am
+++ b/tests/tools/health/Makefile.am
@@ -10,6 +10,12 @@ endif
 
 UTILS=
 
+lib_LTLIBRARIES=libhealthexit.la
+
+# Health thread exit ld_preloaded test lib
+libhealthexit_la_SOURCES=health_exit.c
+libhealthexit_la_LDFLAGS= -module
+
 noinst_PROGRAMS = health_check
 
 health_check_SOURCES = health_check.c $(UTILS)
diff --git a/tests/tools/health/health_exit.c b/tests/tools/health/health_exit.c
new file mode 100644
index 0000000..c2382f2
--- /dev/null
+++ b/tests/tools/health/health_exit.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright (C) 2012 - Christian Babeux <christian.babeux@efficios.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License, version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+
+/*
+ * Check if the specified environment variable is set.
+ * Return 1 if set, otherwise 0.
+ */
+int check_env_var(const char *env)
+{
+	if (env) {
+		if (getenv(env) != NULL && (strncmp(env_val, "1", 1) == 0)) {
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+void __testpoint_thread_manage_clients(void)
+{
+	const char *var = "LTTNG_THREAD_MANAGE_CLIENTS_EXIT";
+
+	if (check_env_var(var)) {
+		pthread_exit(NULL);
+	}
+}
+
+void __testpoint_thread_registration_apps(void)
+{
+	const char *var = "LTTNG_THREAD_REG_APPS_EXIT";
+
+	if (check_env_var(var)) {
+		pthread_exit(NULL);
+	}
+}
+
+void __testpoint_thread_manage_apps(void)
+{
+	const char *var = "LTTNG_THREAD_MANAGE_APPS_EXIT";
+
+	if (check_env_var(var)) {
+		pthread_exit(NULL);
+	}
+}
+
+void __testpoint_thread_manage_kernel(void)
+{
+	const char *var = "LTTNG_THREAD_MANAGE_KERNEL_EXIT";
+
+	if (check_env_var(var)) {
+		pthread_exit(NULL);
+	}
+}
+
+void __testpoint_thread_manage_consumer(void)
+{
+	const char *var = "LTTNG_THREAD_MANAGE_CONSUMER_EXIT";
+
+	if (check_env_var(var)) {
+		pthread_exit(NULL);
+	}
+}
diff --git a/tests/tools/health/health_thread_exit b/tests/tools/health/health_thread_exit
new file mode 100755
index 0000000..dab6b64
--- /dev/null
+++ b/tests/tools/health/health_thread_exit
@@ -0,0 +1,105 @@
+#!/bin/bash
+#
+# Copyright (C) - 2012 Christian Babeux <christian.babeux@efficios.com>
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License, version 2 only, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+TEST_DESC="Health check - Thread exit"
+
+CURDIR=$(dirname $0)/
+TESTDIR=$CURDIR/../..
+LTTNG_BIN="lttng"
+SESSION_NAME="health_thread_exit"
+EVENT_NAME="bogus"
+HEALTH_CHECK_BIN="health_check"
+SESSIOND_PRELOAD=".libs/libhealthexit.so"
+
+source $TESTDIR/utils.sh
+
+print_test_banner "$TEST_DESC"
+
+if [ ! -f "$SESSIOND_PRELOAD" ]; then
+	echo -e "libhealthexit.so not available for this test. Skipping."
+	exit 0
+fi
+
+function test_thread_exit
+{
+	test_thread_exit_name="$1"
+	test_thread_exit_code="$2"
+
+	echo ""
+	echo -e "=== Testing health failure with ${test_thread_exit_name}"
+
+	# Activate testpoints
+	export LTTNG_TESTPOINT_ENABLE=1
+
+	# Activate specific thread exit
+	export ${test_thread_exit_name}_EXIT=1
+
+	# Spawn sessiond with preload healthexit lib
+	export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
+	start_lttng_sessiond
+
+	# Cleanup some env. var.
+	unset LD_PRELOAD
+	unset ${test_thread_exit_name}_EXIT
+
+	# Check initial health status
+	$CURDIR/$HEALTH_CHECK_BIN &> /dev/null
+
+	echo -n "Validating thread ${test_thread_exit_name} failure... "
+
+	# Wait
+	sleep 25
+
+	# Check health status, exit code should indicate failure
+	$CURDIR/$HEALTH_CHECK_BIN &> /dev/null
+
+	health_check_exit_code=$?
+
+	if [ $health_check_exit_code -eq $test_thread_exit_code ]; then
+		print_ok
+		stop_lttng_sessiond
+	else
+		print_fail
+		echo -e "Health returned: $health_check_exit_code\n"
+
+		stop_lttng_sessiond
+		return 1
+	fi
+}
+
+THREAD=("LTTNG_THREAD_MANAGE_CLIENTS"
+	"LTTNG_THREAD_MANAGE_APPS"
+	"LTTNG_THREAD_REG_APPS"
+	"LTTNG_THREAD_MANAGE_KERNEL")
+
+# Exit code value to indicate specific thread failure
+EXIT_CODE=(1 2 4 8)
+
+THREAD_COUNT=${#THREAD[@]}
+i=0
+while [ "$i" -lt "$THREAD_COUNT" ]; do
+	test_thread_exit "${THREAD[$i]}" "${EXIT_CODE[$i]}"
+
+	if [ $? -eq 1 ]; then
+		exit 1
+	fi
+
+	let "i++"
+done
+
+# Special case manage consumer, need to spawn consumer via commands.
+#"LTTNG_THREAD_MANAGE_CONSUMER"
-- 
1.7.12

[PATCH lttng-tools 4/5] Tests: Add health check thread stall test

[Christian Babeux]

This test trigger a "code stall" in a specified thread using the
testpoint mechanism. The testpoint behavior is implemented in
health_stall.c. The testpoint code stall a specific thread processing
by calling sleep(3).

The test select the thread to be stalled by enabling a specific
environment variable.

The test ensure the threads can be succesfully stalled and that the
health check feature is able to properly detect stalling in non-polling
cases.

Signed-off-by: Christian Babeux <christian.babeux@efficios.com>
---
 tests/tools/health/Makefile.am         |   6 +-
 tests/tools/health/health_stall.c      |  66 +++++++++++++++++
 tests/tools/health/health_thread_stall | 128 +++++++++++++++++++++++++++++++++
 3 files changed, 199 insertions(+), 1 deletion(-)
 create mode 100644 tests/tools/health/health_stall.c
 create mode 100755 tests/tools/health/health_thread_stall

diff --git a/tests/tools/health/Makefile.am b/tests/tools/health/Makefile.am
index 0a3f6c5..9fab582 100644
--- a/tests/tools/health/Makefile.am
+++ b/tests/tools/health/Makefile.am
@@ -10,12 +10,16 @@ endif
 
 UTILS=
 
-lib_LTLIBRARIES=libhealthexit.la
+lib_LTLIBRARIES=libhealthexit.la libhealthstall.la
 
 # Health thread exit ld_preloaded test lib
 libhealthexit_la_SOURCES=health_exit.c
 libhealthexit_la_LDFLAGS= -module
 
+# Health thread stall ld_preloaded test lib
+libhealthstall_la_SOURCES=health_stall.c
+libhealthstall_la_LDFLAGS= -module
+
 noinst_PROGRAMS = health_check
 
 health_check_SOURCES = health_check.c $(UTILS)
diff --git a/tests/tools/health/health_stall.c b/tests/tools/health/health_stall.c
new file mode 100644
index 0000000..86b6986
--- /dev/null
+++ b/tests/tools/health/health_stall.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2012 - Christian Babeux <christian.babeux@efficios.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License, version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#define STALL_TIME 60
+
+/*
+ * Check if the specified environment variable is set.
+ * Return 1 if set, otherwise 0.
+ */
+int check_env_var(const char *env)
+{
+	if (env) {
+		if (getenv(env) != NULL && (strncmp(env_val, "1", 1) == 0)) {
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+void __testpoint_thread_manage_clients_before_loop(void)
+{
+	const char *var = "LTTNG_THREAD_MANAGE_CLIENTS_STALL";
+
+	if (check_env_var(var)) {
+		sleep(STALL_TIME);
+	}
+}
+
+void __testpoint_thread_manage_kernel_before_loop(void)
+{
+	const char *var = "LTTNG_THREAD_MANAGE_KERNEL_STALL";
+
+	if (check_env_var(var)) {
+		sleep(STALL_TIME);
+	}
+}
+
+void __testpoint_thread_manage_apps_before_loop(void)
+{
+	const char *var = "LTTNG_THREAD_MANAGE_APPS_STALL";
+
+	if (check_env_var(var)) {
+		sleep(STALL_TIME);
+	}
+}
+
diff --git a/tests/tools/health/health_thread_stall b/tests/tools/health/health_thread_stall
new file mode 100755
index 0000000..d870895
--- /dev/null
+++ b/tests/tools/health/health_thread_stall
@@ -0,0 +1,128 @@
+#!/bin/bash
+#
+# Copyright (C) - 2012 Christian Babeux <christian.babeux@efficios.com>
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License, version 2 only, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+TEST_DESC="Health check - Thread stall"
+
+CURDIR=$(dirname $0)/
+TESTDIR=$CURDIR/../..
+LTTNG_BIN="lttng"
+SESSION_NAME="health_thread_stall"
+EVENT_NAME="bogus"
+HEALTH_CHECK_BIN="health_check"
+SESSIOND_PRELOAD=".libs/libhealthstall.so"
+
+source $TESTDIR/utils.sh
+
+print_test_banner "$TEST_DESC"
+
+if [ ! -f "$SESSIOND_PRELOAD" ]; then
+	echo -e "libhealthstall.so not available for this test. Skipping."
+	exit 0
+fi
+
+function test_thread_stall
+{
+	test_thread_stall_name="$1"
+	test_thread_exit_code="$2"
+
+	echo ""
+	echo -e "=== Testing health failure with ${test_thread_stall_name}"
+
+	# Activate testpoints
+	export LTTNG_TESTPOINT_ENABLE=1
+
+	# Activate specific thread exit
+	export ${test_thread_stall_name}_STALL=1
+
+	# Spawn sessiond with preload healthexit lib
+	export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
+	start_lttng_sessiond
+
+	# Cleanup some env. var.
+	unset LD_PRELOAD
+	unset ${test_thread_stall_name}_STALL
+
+	# Check initial health status
+	$CURDIR/$HEALTH_CHECK_BIN &> /dev/null
+
+	echo -n "Validating that ${test_thread_stall_name} is stalled... "
+
+	# Wait
+	sleep 25
+
+	# Check health status, exit code should indicate failure
+	$CURDIR/$HEALTH_CHECK_BIN &> /dev/null
+
+	health_check_exit_code=$?
+
+	if [ $health_check_exit_code -eq $test_thread_exit_code ]; then
+		print_ok
+	else
+		print_fail
+		echo -e "Health returned: $health_check_exit_code\n"
+
+		stop_lttng_sessiond
+		return 1
+	fi
+
+	echo -n "Validating that ${test_thread_stall_name} is no longer stalled... "
+
+	# Wait
+	sleep 40
+
+	# Check health status, exit code should now pass
+	$CURDIR/$HEALTH_CHECK_BIN &> /dev/null
+
+	health_check_exit_code=$?
+
+	if [ $health_check_exit_code -eq 0 ]; then
+		print_ok
+		stop_lttng_sessiond
+	else
+		print_fail
+		echo -e "Health returned: $health_check_exit_code\n"
+		stop_lttng_sessiond
+		return 1
+	fi
+
+
+}
+
+THREAD=("LTTNG_THREAD_MANAGE_CLIENTS"
+	"LTTNG_THREAD_MANAGE_APPS"
+# This thread is a little bit tricky to stall,
+# need to send some commands and setup an app.
+#	"LTTNG_THREAD_REG_APPS"
+	"LTTNG_THREAD_MANAGE_KERNEL")
+
+# Exit code value to indicate specific thread failure
+EXIT_CODE=(1
+	   2
+#	   4
+	   8)
+
+THREAD_COUNT=${#THREAD[@]}
+i=0
+while [ "$i" -lt "$THREAD_COUNT" ]; do
+	test_thread_stall "${THREAD[$i]}" "${EXIT_CODE[$i]}"
+
+	if [ $? -eq 1 ]; then
+		exit 1
+	fi
+
+	let "i++"
+done
-- 
1.7.12

Re: [PATCH lttng-tools 2/5] Tests: Add a health check utility program

[Mathieu Desnoyers]

* Christian Babeux (christian.babeux@efficios.com) wrote:
> The health_check program is a simple utility to query the health
> status of the different threads of the sessiond.

do we want to call it lttng-health-check and install it in the system ?

Mathieu

> 
> Sample output:
> 
> > ./health_check
> Health check cmd: 0
> Health check app. manage: 0
> Health check app. registration: 0
> Health check kernel: 0
> Health check consumer: 0
> 
> The return code is encoded to indicate which thread has failed.
> 
> Signed-off-by: Christian Babeux <christian.babeux@efficios.com>
> ---
>  tests/tools/health/Makefile.am    | 20 +++++++++++
>  tests/tools/health/health_check.c | 73 +++++++++++++++++++++++++++++++++++++++
>  2 files changed, 93 insertions(+)
>  create mode 100644 tests/tools/health/Makefile.am
>  create mode 100644 tests/tools/health/health_check.c
> 
> diff --git a/tests/tools/health/Makefile.am b/tests/tools/health/Makefile.am
> new file mode 100644
> index 0000000..09573db
> --- /dev/null
> +++ b/tests/tools/health/Makefile.am
> @@ -0,0 +1,20 @@
> +AM_CFLAGS = -I. -O2 -g -I../../../include
> +AM_LDFLAGS =
> +
> +if LTTNG_TOOLS_BUILD_WITH_LIBDL
> +AM_LDFLAGS += -ldl
> +endif
> +if LTTNG_TOOLS_BUILD_WITH_LIBC_DL
> +AM_LDFLAGS += -lc
> +endif
> +
> +UTILS=
> +
> +noinst_PROGRAMS = health_check
> +
> +health_check_SOURCES = health_check.c $(UTILS)
> +health_check_LDADD = $(top_builddir)/src/lib/lttng-ctl/liblttng-ctl.la \
> +		     $(top_builddir)/src/common/libcommon.la
> +
> +noinst_SCRIPTS =
> +EXTRA_DIST =
> diff --git a/tests/tools/health/health_check.c b/tests/tools/health/health_check.c
> new file mode 100644
> index 0000000..3eef110
> --- /dev/null
> +++ b/tests/tools/health/health_check.c
> @@ -0,0 +1,73 @@
> +/*
> + * Copyright (C) 2012 - Christian Babeux <christian.babeux@efficios.com>
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms of the GNU General Public License, version 2 only, as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful, but WITHOUT
> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> + * more details.
> + *
> + * You should have received a copy of the GNU General Public License along with
> + * this program; if not, write to the Free Software Foundation, Inc., 51
> + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> + */
> +
> +#include <stdio.h>
> +
> +#include "lttng/lttng.h"
> +
> +#define HEALTH_CMD_FAIL     (1 << 0)
> +#define HEALTH_APP_MNG_FAIL (1 << 1)
> +#define HEALTH_APP_REG_FAIL (1 << 2)
> +#define HEALTH_KERNEL_FAIL  (1 << 3)
> +#define HEALTH_CSMR_FAIL    (1 << 4)
> +
> +int main(int argc, char *argv[])
> +{
> +	int health = -1;
> +	int status = 0;
> +
> +	/* Command thread */
> +	health = lttng_health_check(LTTNG_HEALTH_CMD);
> +	printf("Health check cmd: %d\n", health);
> +
> +	if (health) {
> +		status |= HEALTH_CMD_FAIL;
> +	}
> +
> +	/* App manage thread */
> +	health = lttng_health_check(LTTNG_HEALTH_APP_MANAGE);
> +	printf("Health check app. manage: %d\n", health);
> +
> +	if (health) {
> +		status |= HEALTH_APP_MNG_FAIL;
> +	}
> +	/* App registration thread */
> +	health = lttng_health_check(LTTNG_HEALTH_APP_REG);
> +	printf("Health check app. registration: %d\n", health);
> +
> +	if (health) {
> +		status |= HEALTH_APP_REG_FAIL;
> +	}
> +
> +	/* Kernel thread */
> +	health = lttng_health_check(LTTNG_HEALTH_KERNEL);
> +	printf("Health check kernel: %d\n", health);
> +
> +	if (health) {
> +		status |= HEALTH_KERNEL_FAIL;
> +	}
> +
> +	/* Consumer thread */
> +	health = lttng_health_check(LTTNG_HEALTH_CONSUMER);
> +	printf("Health check consumer: %d\n", health);
> +
> +	if (health) {
> +		status |= HEALTH_CSMR_FAIL;
> +	}
> +
> +	return status;
> +}
> -- 
> 1.7.12
> 
> 
> _______________________________________________
> lttng-dev mailing list
> lttng-dev@lists.lttng.org
> http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

-- 
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

Re: [PATCH lttng-tools 4/5] Tests: Add health check thread stall test

[Mathieu Desnoyers]

* Christian Babeux (christian.babeux@efficios.com) wrote:
> This test trigger a "code stall" in a specified thread using the
> testpoint mechanism. The testpoint behavior is implemented in
> health_stall.c. The testpoint code stall a specific thread processing
> by calling sleep(3).
> 
> The test select the thread to be stalled by enabling a specific
> environment variable.
> 
> The test ensure the threads can be succesfully stalled and that the
> health check feature is able to properly detect stalling in non-polling
> cases.
> 
> Signed-off-by: Christian Babeux <christian.babeux@efficios.com>
> ---
>  tests/tools/health/Makefile.am         |   6 +-
>  tests/tools/health/health_stall.c      |  66 +++++++++++++++++
>  tests/tools/health/health_thread_stall | 128 +++++++++++++++++++++++++++++++++
>  3 files changed, 199 insertions(+), 1 deletion(-)
>  create mode 100644 tests/tools/health/health_stall.c
>  create mode 100755 tests/tools/health/health_thread_stall
> 
> diff --git a/tests/tools/health/Makefile.am b/tests/tools/health/Makefile.am
> index 0a3f6c5..9fab582 100644
> --- a/tests/tools/health/Makefile.am
> +++ b/tests/tools/health/Makefile.am
> @@ -10,12 +10,16 @@ endif
>  
>  UTILS=
>  
> -lib_LTLIBRARIES=libhealthexit.la
> +lib_LTLIBRARIES=libhealthexit.la libhealthstall.la
>  
>  # Health thread exit ld_preloaded test lib
>  libhealthexit_la_SOURCES=health_exit.c
>  libhealthexit_la_LDFLAGS= -module
>  
> +# Health thread stall ld_preloaded test lib
> +libhealthstall_la_SOURCES=health_stall.c
> +libhealthstall_la_LDFLAGS= -module
> +
>  noinst_PROGRAMS = health_check
>  
>  health_check_SOURCES = health_check.c $(UTILS)
> diff --git a/tests/tools/health/health_stall.c b/tests/tools/health/health_stall.c
> new file mode 100644
> index 0000000..86b6986
> --- /dev/null
> +++ b/tests/tools/health/health_stall.c
> @@ -0,0 +1,66 @@
> +/*
> + * Copyright (C) 2012 - Christian Babeux <christian.babeux@efficios.com>
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms of the GNU General Public License, version 2 only, as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful, but WITHOUT
> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> + * more details.
> + *
> + * You should have received a copy of the GNU General Public License along with
> + * this program; if not, write to the Free Software Foundation, Inc., 51
> + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> + */
> +
> +#include <stdlib.h>
> +#include <string.h>
> +#include <stdio.h>
> +#include <unistd.h>
> +
> +#define STALL_TIME 60

could we introduce a LTTNG_HEALTH_CHECK_STALL env. var ?

Thanks,

Mathieu

> +
> +/*
> + * Check if the specified environment variable is set.
> + * Return 1 if set, otherwise 0.
> + */
> +int check_env_var(const char *env)
> +{
> +	if (env) {
> +		if (getenv(env) != NULL && (strncmp(env_val, "1", 1) == 0)) {
> +			return 1;
> +		}
> +	}
> +
> +	return 0;
> +}
> +
> +void __testpoint_thread_manage_clients_before_loop(void)
> +{
> +	const char *var = "LTTNG_THREAD_MANAGE_CLIENTS_STALL";
> +
> +	if (check_env_var(var)) {
> +		sleep(STALL_TIME);
> +	}
> +}
> +
> +void __testpoint_thread_manage_kernel_before_loop(void)
> +{
> +	const char *var = "LTTNG_THREAD_MANAGE_KERNEL_STALL";
> +
> +	if (check_env_var(var)) {
> +		sleep(STALL_TIME);
> +	}
> +}
> +
> +void __testpoint_thread_manage_apps_before_loop(void)
> +{
> +	const char *var = "LTTNG_THREAD_MANAGE_APPS_STALL";
> +
> +	if (check_env_var(var)) {
> +		sleep(STALL_TIME);
> +	}
> +}
> +
> diff --git a/tests/tools/health/health_thread_stall b/tests/tools/health/health_thread_stall
> new file mode 100755
> index 0000000..d870895
> --- /dev/null
> +++ b/tests/tools/health/health_thread_stall
> @@ -0,0 +1,128 @@
> +#!/bin/bash
> +#
> +# Copyright (C) - 2012 Christian Babeux <christian.babeux@efficios.com>
> +#
> +# This program is free software; you can redistribute it and/or modify it
> +# under the terms of the GNU General Public License, version 2 only, as
> +# published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it will be useful, but WITHOUT
> +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> +# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> +# more details.
> +#
> +# You should have received a copy of the GNU General Public License along with
> +# this program; if not, write to the Free Software Foundation, Inc., 51
> +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> +
> +TEST_DESC="Health check - Thread stall"
> +
> +CURDIR=$(dirname $0)/
> +TESTDIR=$CURDIR/../..
> +LTTNG_BIN="lttng"
> +SESSION_NAME="health_thread_stall"
> +EVENT_NAME="bogus"
> +HEALTH_CHECK_BIN="health_check"
> +SESSIOND_PRELOAD=".libs/libhealthstall.so"
> +
> +source $TESTDIR/utils.sh
> +
> +print_test_banner "$TEST_DESC"
> +
> +if [ ! -f "$SESSIOND_PRELOAD" ]; then
> +	echo -e "libhealthstall.so not available for this test. Skipping."
> +	exit 0
> +fi
> +
> +function test_thread_stall
> +{
> +	test_thread_stall_name="$1"
> +	test_thread_exit_code="$2"
> +
> +	echo ""
> +	echo -e "=== Testing health failure with ${test_thread_stall_name}"
> +
> +	# Activate testpoints
> +	export LTTNG_TESTPOINT_ENABLE=1
> +
> +	# Activate specific thread exit
> +	export ${test_thread_stall_name}_STALL=1
> +
> +	# Spawn sessiond with preload healthexit lib
> +	export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
> +	start_lttng_sessiond
> +
> +	# Cleanup some env. var.
> +	unset LD_PRELOAD
> +	unset ${test_thread_stall_name}_STALL
> +
> +	# Check initial health status
> +	$CURDIR/$HEALTH_CHECK_BIN &> /dev/null
> +
> +	echo -n "Validating that ${test_thread_stall_name} is stalled... "
> +
> +	# Wait
> +	sleep 25
> +
> +	# Check health status, exit code should indicate failure
> +	$CURDIR/$HEALTH_CHECK_BIN &> /dev/null
> +
> +	health_check_exit_code=$?
> +
> +	if [ $health_check_exit_code -eq $test_thread_exit_code ]; then
> +		print_ok
> +	else
> +		print_fail
> +		echo -e "Health returned: $health_check_exit_code\n"
> +
> +		stop_lttng_sessiond
> +		return 1
> +	fi
> +
> +	echo -n "Validating that ${test_thread_stall_name} is no longer stalled... "
> +
> +	# Wait
> +	sleep 40
> +
> +	# Check health status, exit code should now pass
> +	$CURDIR/$HEALTH_CHECK_BIN &> /dev/null
> +
> +	health_check_exit_code=$?
> +
> +	if [ $health_check_exit_code -eq 0 ]; then
> +		print_ok
> +		stop_lttng_sessiond
> +	else
> +		print_fail
> +		echo -e "Health returned: $health_check_exit_code\n"
> +		stop_lttng_sessiond
> +		return 1
> +	fi
> +
> +
> +}
> +
> +THREAD=("LTTNG_THREAD_MANAGE_CLIENTS"
> +	"LTTNG_THREAD_MANAGE_APPS"
> +# This thread is a little bit tricky to stall,
> +# need to send some commands and setup an app.
> +#	"LTTNG_THREAD_REG_APPS"
> +	"LTTNG_THREAD_MANAGE_KERNEL")
> +
> +# Exit code value to indicate specific thread failure
> +EXIT_CODE=(1
> +	   2
> +#	   4
> +	   8)
> +
> +THREAD_COUNT=${#THREAD[@]}
> +i=0
> +while [ "$i" -lt "$THREAD_COUNT" ]; do
> +	test_thread_stall "${THREAD[$i]}" "${EXIT_CODE[$i]}"
> +
> +	if [ $? -eq 1 ]; then
> +		exit 1
> +	fi
> +
> +	let "i++"
> +done
> -- 
> 1.7.12
> 
> 
> _______________________________________________
> lttng-dev mailing list
> lttng-dev@lists.lttng.org
> http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

-- 
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

Re: [PATCH lttng-tools 2/5] Tests: Add a health check utility program

[David Goulet]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA512

<Mathieu Desnoyers:
> * Christian Babeux (christian.babeux@efficios.com) wrote:
>> The health_check program is a simple utility to query the health 
>> status of the different threads of the sessiond.
> 
> do we want to call it lttng-health-check and install it in the
> system ?
> 

We could but I would really prefer add a "lttng health" that calls
this or merge it as a lttng command. I'm no fan of multiplying
binaries :).

David

> Mathieu
> 
-----BEGIN PGP SIGNATURE-----

iQEcBAEBCgAGBQJQZL1xAAoJEELoaioR9I02wbgH/j/Qs0my1MzvVvOWtmph08Ll
UKfpGFOXsyHgl96dKNuTzqQ8iSZIDHULCkwHCMDCkRRLiXJ7pyCGN/GkU1oCvLmD
Rc+nfs2MotUZtoqD1NicCUf2d8VIc37pKU0ImeDZMuWk+3lUIqY1//gcUsVdAYJL
18fkIo5ywr7oIdaZdKTRAuXI0ZesZuChpT7mltbcfnfHAp8viIzvPKuvr48V10sE
/jVyYM5tvPvAoCfO15uosTV+m9+/R2taxF1E+Ng5obZtWUurRCPqw28tOW+W9hq4
+Lbwg814ePsVDfOItBjxwXiWhuElGP6SnizD+yUB7ca7Eadcq/Yodb9DvayAtQk=
=4p87
-----END PGP SIGNATURE-----

Re: [PATCH lttng-tools 4/5] Tests: Add health check thread stall test

[Christian Babeux]

Hi,

Please disregard this patch because it's missing the env_val
declaration. A v2 will soon follow.

Thank you,

Christian

On Thu, Sep 27, 2012 at 2:23 PM, Christian Babeux
<christian.babeux@efficios.com> wrote:
> This test trigger a "code stall" in a specified thread using the
> testpoint mechanism. The testpoint behavior is implemented in
> health_stall.c. The testpoint code stall a specific thread processing
> by calling sleep(3).
>
> The test select the thread to be stalled by enabling a specific
> environment variable.
>
> The test ensure the threads can be succesfully stalled and that the
> health check feature is able to properly detect stalling in non-polling
> cases.
>
> Signed-off-by: Christian Babeux <christian.babeux@efficios.com>
> ---
>  tests/tools/health/Makefile.am         |   6 +-
>  tests/tools/health/health_stall.c      |  66 +++++++++++++++++
>  tests/tools/health/health_thread_stall | 128 +++++++++++++++++++++++++++++++++
>  3 files changed, 199 insertions(+), 1 deletion(-)
>  create mode 100644 tests/tools/health/health_stall.c
>  create mode 100755 tests/tools/health/health_thread_stall
>
> diff --git a/tests/tools/health/Makefile.am b/tests/tools/health/Makefile.am
> index 0a3f6c5..9fab582 100644
> --- a/tests/tools/health/Makefile.am
> +++ b/tests/tools/health/Makefile.am
> @@ -10,12 +10,16 @@ endif
>
>  UTILS=
>
> -lib_LTLIBRARIES=libhealthexit.la
> +lib_LTLIBRARIES=libhealthexit.la libhealthstall.la
>
>  # Health thread exit ld_preloaded test lib
>  libhealthexit_la_SOURCES=health_exit.c
>  libhealthexit_la_LDFLAGS= -module
>
> +# Health thread stall ld_preloaded test lib
> +libhealthstall_la_SOURCES=health_stall.c
> +libhealthstall_la_LDFLAGS= -module
> +
>  noinst_PROGRAMS = health_check
>
>  health_check_SOURCES = health_check.c $(UTILS)
> diff --git a/tests/tools/health/health_stall.c b/tests/tools/health/health_stall.c
> new file mode 100644
> index 0000000..86b6986
> --- /dev/null
> +++ b/tests/tools/health/health_stall.c
> @@ -0,0 +1,66 @@
> +/*
> + * Copyright (C) 2012 - Christian Babeux <christian.babeux@efficios.com>
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms of the GNU General Public License, version 2 only, as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful, but WITHOUT
> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> + * more details.
> + *
> + * You should have received a copy of the GNU General Public License along with
> + * this program; if not, write to the Free Software Foundation, Inc., 51
> + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> + */
> +
> +#include <stdlib.h>
> +#include <string.h>
> +#include <stdio.h>
> +#include <unistd.h>
> +
> +#define STALL_TIME 60
> +
> +/*
> + * Check if the specified environment variable is set.
> + * Return 1 if set, otherwise 0.
> + */
> +int check_env_var(const char *env)
> +{
> +       if (env) {
> +               if (getenv(env) != NULL && (strncmp(env_val, "1", 1) == 0)) {
> +                       return 1;
> +               }
> +       }
> +
> +       return 0;
> +}
> +
> +void __testpoint_thread_manage_clients_before_loop(void)
> +{
> +       const char *var = "LTTNG_THREAD_MANAGE_CLIENTS_STALL";
> +
> +       if (check_env_var(var)) {
> +               sleep(STALL_TIME);
> +       }
> +}
> +
> +void __testpoint_thread_manage_kernel_before_loop(void)
> +{
> +       const char *var = "LTTNG_THREAD_MANAGE_KERNEL_STALL";
> +
> +       if (check_env_var(var)) {
> +               sleep(STALL_TIME);
> +       }
> +}
> +
> +void __testpoint_thread_manage_apps_before_loop(void)
> +{
> +       const char *var = "LTTNG_THREAD_MANAGE_APPS_STALL";
> +
> +       if (check_env_var(var)) {
> +               sleep(STALL_TIME);
> +       }
> +}
> +
> diff --git a/tests/tools/health/health_thread_stall b/tests/tools/health/health_thread_stall
> new file mode 100755
> index 0000000..d870895
> --- /dev/null
> +++ b/tests/tools/health/health_thread_stall
> @@ -0,0 +1,128 @@
> +#!/bin/bash
> +#
> +# Copyright (C) - 2012 Christian Babeux <christian.babeux@efficios.com>
> +#
> +# This program is free software; you can redistribute it and/or modify it
> +# under the terms of the GNU General Public License, version 2 only, as
> +# published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it will be useful, but WITHOUT
> +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> +# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> +# more details.
> +#
> +# You should have received a copy of the GNU General Public License along with
> +# this program; if not, write to the Free Software Foundation, Inc., 51
> +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> +
> +TEST_DESC="Health check - Thread stall"
> +
> +CURDIR=$(dirname $0)/
> +TESTDIR=$CURDIR/../..
> +LTTNG_BIN="lttng"
> +SESSION_NAME="health_thread_stall"
> +EVENT_NAME="bogus"
> +HEALTH_CHECK_BIN="health_check"
> +SESSIOND_PRELOAD=".libs/libhealthstall.so"
> +
> +source $TESTDIR/utils.sh
> +
> +print_test_banner "$TEST_DESC"
> +
> +if [ ! -f "$SESSIOND_PRELOAD" ]; then
> +       echo -e "libhealthstall.so not available for this test. Skipping."
> +       exit 0
> +fi
> +
> +function test_thread_stall
> +{
> +       test_thread_stall_name="$1"
> +       test_thread_exit_code="$2"
> +
> +       echo ""
> +       echo -e "=== Testing health failure with ${test_thread_stall_name}"
> +
> +       # Activate testpoints
> +       export LTTNG_TESTPOINT_ENABLE=1
> +
> +       # Activate specific thread exit
> +       export ${test_thread_stall_name}_STALL=1
> +
> +       # Spawn sessiond with preload healthexit lib
> +       export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
> +       start_lttng_sessiond
> +
> +       # Cleanup some env. var.
> +       unset LD_PRELOAD
> +       unset ${test_thread_stall_name}_STALL
> +
> +       # Check initial health status
> +       $CURDIR/$HEALTH_CHECK_BIN &> /dev/null
> +
> +       echo -n "Validating that ${test_thread_stall_name} is stalled... "
> +
> +       # Wait
> +       sleep 25
> +
> +       # Check health status, exit code should indicate failure
> +       $CURDIR/$HEALTH_CHECK_BIN &> /dev/null
> +
> +       health_check_exit_code=$?
> +
> +       if [ $health_check_exit_code -eq $test_thread_exit_code ]; then
> +               print_ok
> +       else
> +               print_fail
> +               echo -e "Health returned: $health_check_exit_code\n"
> +
> +               stop_lttng_sessiond
> +               return 1
> +       fi
> +
> +       echo -n "Validating that ${test_thread_stall_name} is no longer stalled... "
> +
> +       # Wait
> +       sleep 40
> +
> +       # Check health status, exit code should now pass
> +       $CURDIR/$HEALTH_CHECK_BIN &> /dev/null
> +
> +       health_check_exit_code=$?
> +
> +       if [ $health_check_exit_code -eq 0 ]; then
> +               print_ok
> +               stop_lttng_sessiond
> +       else
> +               print_fail
> +               echo -e "Health returned: $health_check_exit_code\n"
> +               stop_lttng_sessiond
> +               return 1
> +       fi
> +
> +
> +}
> +
> +THREAD=("LTTNG_THREAD_MANAGE_CLIENTS"
> +       "LTTNG_THREAD_MANAGE_APPS"
> +# This thread is a little bit tricky to stall,
> +# need to send some commands and setup an app.
> +#      "LTTNG_THREAD_REG_APPS"
> +       "LTTNG_THREAD_MANAGE_KERNEL")
> +
> +# Exit code value to indicate specific thread failure
> +EXIT_CODE=(1
> +          2
> +#         4
> +          8)
> +
> +THREAD_COUNT=${#THREAD[@]}
> +i=0
> +while [ "$i" -lt "$THREAD_COUNT" ]; do
> +       test_thread_stall "${THREAD[$i]}" "${EXIT_CODE[$i]}"
> +
> +       if [ $? -eq 1 ]; then
> +               exit 1
> +       fi
> +
> +       let "i++"
> +done
> --
> 1.7.12
>

Re: [PATCH lttng-tools 3/5] Tests: Add health check thread exit test

[Christian Babeux]

Hi,

Please disregard this patch because it's missing the env_val
declaration. A v2 will soon follow.

Thank you,

Christian

On Thu, Sep 27, 2012 at 2:23 PM, Christian Babeux
<christian.babeux@efficios.com> wrote:
> This test trigger a failure in a specified thread using the
> recently introduced testpoint mechanism. The testpoints behavior
> is implemented in health_exit.c. The testpoint code simply calls
> pthread_exit(3) and effectively "kill" the thread without affecting
> the other threads behavior.
>
> The test select the thread to be "killed" by enabling a specific
> environment variable.
>
> With this test we ensure that each thread can be succesfully terminated
> and that the health check feature properly detect a failure.
>
> Signed-off-by: Christian Babeux <christian.babeux@efficios.com>
> ---
>  tests/tools/health/Makefile.am        |   6 ++
>  tests/tools/health/health_exit.c      |  80 ++++++++++++++++++++++++++
>  tests/tools/health/health_thread_exit | 105 ++++++++++++++++++++++++++++++++++
>  3 files changed, 191 insertions(+)
>  create mode 100644 tests/tools/health/health_exit.c
>  create mode 100755 tests/tools/health/health_thread_exit
>
> diff --git a/tests/tools/health/Makefile.am b/tests/tools/health/Makefile.am
> index 09573db..0a3f6c5 100644
> --- a/tests/tools/health/Makefile.am
> +++ b/tests/tools/health/Makefile.am
> @@ -10,6 +10,12 @@ endif
>
>  UTILS=
>
> +lib_LTLIBRARIES=libhealthexit.la
> +
> +# Health thread exit ld_preloaded test lib
> +libhealthexit_la_SOURCES=health_exit.c
> +libhealthexit_la_LDFLAGS= -module
> +
>  noinst_PROGRAMS = health_check
>
>  health_check_SOURCES = health_check.c $(UTILS)
> diff --git a/tests/tools/health/health_exit.c b/tests/tools/health/health_exit.c
> new file mode 100644
> index 0000000..c2382f2
> --- /dev/null
> +++ b/tests/tools/health/health_exit.c
> @@ -0,0 +1,80 @@
> +/*
> + * Copyright (C) 2012 - Christian Babeux <christian.babeux@efficios.com>
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms of the GNU General Public License, version 2 only, as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful, but WITHOUT
> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> + * more details.
> + *
> + * You should have received a copy of the GNU General Public License along with
> + * this program; if not, write to the Free Software Foundation, Inc., 51
> + * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> + */
> +
> +#include <stdlib.h>
> +#include <string.h>
> +#include <pthread.h>
> +
> +/*
> + * Check if the specified environment variable is set.
> + * Return 1 if set, otherwise 0.
> + */
> +int check_env_var(const char *env)
> +{
> +       if (env) {
> +               if (getenv(env) != NULL && (strncmp(env_val, "1", 1) == 0)) {
> +                       return 1;
> +               }
> +       }
> +
> +       return 0;
> +}
> +
> +void __testpoint_thread_manage_clients(void)
> +{
> +       const char *var = "LTTNG_THREAD_MANAGE_CLIENTS_EXIT";
> +
> +       if (check_env_var(var)) {
> +               pthread_exit(NULL);
> +       }
> +}
> +
> +void __testpoint_thread_registration_apps(void)
> +{
> +       const char *var = "LTTNG_THREAD_REG_APPS_EXIT";
> +
> +       if (check_env_var(var)) {
> +               pthread_exit(NULL);
> +       }
> +}
> +
> +void __testpoint_thread_manage_apps(void)
> +{
> +       const char *var = "LTTNG_THREAD_MANAGE_APPS_EXIT";
> +
> +       if (check_env_var(var)) {
> +               pthread_exit(NULL);
> +       }
> +}
> +
> +void __testpoint_thread_manage_kernel(void)
> +{
> +       const char *var = "LTTNG_THREAD_MANAGE_KERNEL_EXIT";
> +
> +       if (check_env_var(var)) {
> +               pthread_exit(NULL);
> +       }
> +}
> +
> +void __testpoint_thread_manage_consumer(void)
> +{
> +       const char *var = "LTTNG_THREAD_MANAGE_CONSUMER_EXIT";
> +
> +       if (check_env_var(var)) {
> +               pthread_exit(NULL);
> +       }
> +}
> diff --git a/tests/tools/health/health_thread_exit b/tests/tools/health/health_thread_exit
> new file mode 100755
> index 0000000..dab6b64
> --- /dev/null
> +++ b/tests/tools/health/health_thread_exit
> @@ -0,0 +1,105 @@
> +#!/bin/bash
> +#
> +# Copyright (C) - 2012 Christian Babeux <christian.babeux@efficios.com>
> +#
> +# This program is free software; you can redistribute it and/or modify it
> +# under the terms of the GNU General Public License, version 2 only, as
> +# published by the Free Software Foundation.
> +#
> +# This program is distributed in the hope that it will be useful, but WITHOUT
> +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> +# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> +# more details.
> +#
> +# You should have received a copy of the GNU General Public License along with
> +# this program; if not, write to the Free Software Foundation, Inc., 51
> +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> +
> +TEST_DESC="Health check - Thread exit"
> +
> +CURDIR=$(dirname $0)/
> +TESTDIR=$CURDIR/../..
> +LTTNG_BIN="lttng"
> +SESSION_NAME="health_thread_exit"
> +EVENT_NAME="bogus"
> +HEALTH_CHECK_BIN="health_check"
> +SESSIOND_PRELOAD=".libs/libhealthexit.so"
> +
> +source $TESTDIR/utils.sh
> +
> +print_test_banner "$TEST_DESC"
> +
> +if [ ! -f "$SESSIOND_PRELOAD" ]; then
> +       echo -e "libhealthexit.so not available for this test. Skipping."
> +       exit 0
> +fi
> +
> +function test_thread_exit
> +{
> +       test_thread_exit_name="$1"
> +       test_thread_exit_code="$2"
> +
> +       echo ""
> +       echo -e "=== Testing health failure with ${test_thread_exit_name}"
> +
> +       # Activate testpoints
> +       export LTTNG_TESTPOINT_ENABLE=1
> +
> +       # Activate specific thread exit
> +       export ${test_thread_exit_name}_EXIT=1
> +
> +       # Spawn sessiond with preload healthexit lib
> +       export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
> +       start_lttng_sessiond
> +
> +       # Cleanup some env. var.
> +       unset LD_PRELOAD
> +       unset ${test_thread_exit_name}_EXIT
> +
> +       # Check initial health status
> +       $CURDIR/$HEALTH_CHECK_BIN &> /dev/null
> +
> +       echo -n "Validating thread ${test_thread_exit_name} failure... "
> +
> +       # Wait
> +       sleep 25
> +
> +       # Check health status, exit code should indicate failure
> +       $CURDIR/$HEALTH_CHECK_BIN &> /dev/null
> +
> +       health_check_exit_code=$?
> +
> +       if [ $health_check_exit_code -eq $test_thread_exit_code ]; then
> +               print_ok
> +               stop_lttng_sessiond
> +       else
> +               print_fail
> +               echo -e "Health returned: $health_check_exit_code\n"
> +
> +               stop_lttng_sessiond
> +               return 1
> +       fi
> +}
> +
> +THREAD=("LTTNG_THREAD_MANAGE_CLIENTS"
> +       "LTTNG_THREAD_MANAGE_APPS"
> +       "LTTNG_THREAD_REG_APPS"
> +       "LTTNG_THREAD_MANAGE_KERNEL")
> +
> +# Exit code value to indicate specific thread failure
> +EXIT_CODE=(1 2 4 8)
> +
> +THREAD_COUNT=${#THREAD[@]}
> +i=0
> +while [ "$i" -lt "$THREAD_COUNT" ]; do
> +       test_thread_exit "${THREAD[$i]}" "${EXIT_CODE[$i]}"
> +
> +       if [ $? -eq 1 ]; then
> +               exit 1
> +       fi
> +
> +       let "i++"
> +done
> +
> +# Special case manage consumer, need to spawn consumer via commands.
> +#"LTTNG_THREAD_MANAGE_CONSUMER"
> --
> 1.7.12
>

[PATCH v2 lttng-tools 3/5] Tests: Add health check thread exit test

[Christian Babeux]

This test trigger a failure in a specified thread using the
recently introduced testpoint mechanism. The testpoints behavior
is implemented in health_exit.c. The testpoint code simply calls
pthread_exit(3) and effectively "kill" the thread without affecting
the other threads behavior.

The test select the thread to be "killed" by enabling a specific
environment variable.

With this test we ensure that each thread can be succesfully terminated
and that the health check feature properly detect a failure.

Signed-off-by: Christian Babeux <christian.babeux@efficios.com>
---
 tests/tools/health/Makefile.am        |   6 ++
 tests/tools/health/health_exit.c      |  81 ++++++++++++++++++++++++++
 tests/tools/health/health_thread_exit | 105 ++++++++++++++++++++++++++++++++++
 3 files changed, 192 insertions(+)
 create mode 100644 tests/tools/health/health_exit.c
 create mode 100755 tests/tools/health/health_thread_exit

diff --git a/tests/tools/health/Makefile.am b/tests/tools/health/Makefile.am
index 09573db..0a3f6c5 100644
--- a/tests/tools/health/Makefile.am
+++ b/tests/tools/health/Makefile.am
@@ -10,6 +10,12 @@ endif
 
 UTILS=
 
+lib_LTLIBRARIES=libhealthexit.la
+
+# Health thread exit ld_preloaded test lib
+libhealthexit_la_SOURCES=health_exit.c
+libhealthexit_la_LDFLAGS= -module
+
 noinst_PROGRAMS = health_check
 
 health_check_SOURCES = health_check.c $(UTILS)
diff --git a/tests/tools/health/health_exit.c b/tests/tools/health/health_exit.c
new file mode 100644
index 0000000..258b08d
--- /dev/null
+++ b/tests/tools/health/health_exit.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2012 - Christian Babeux <christian.babeux@efficios.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License, version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+
+/*
+ * Check if the specified environment variable is set.
+ * Return 1 if set, otherwise 0.
+ */
+int check_env_var(const char *env)
+{
+	if (env) {
+		char *env_val = getenv(env);
+		if (env_val && (strncmp(env_val, "1", 1) == 0)) {
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+void __testpoint_thread_manage_clients(void)
+{
+	const char *var = "LTTNG_THREAD_MANAGE_CLIENTS_EXIT";
+
+	if (check_env_var(var)) {
+		pthread_exit(NULL);
+	}
+}
+
+void __testpoint_thread_registration_apps(void)
+{
+	const char *var = "LTTNG_THREAD_REG_APPS_EXIT";
+
+	if (check_env_var(var)) {
+		pthread_exit(NULL);
+	}
+}
+
+void __testpoint_thread_manage_apps(void)
+{
+	const char *var = "LTTNG_THREAD_MANAGE_APPS_EXIT";
+
+	if (check_env_var(var)) {
+		pthread_exit(NULL);
+	}
+}
+
+void __testpoint_thread_manage_kernel(void)
+{
+	const char *var = "LTTNG_THREAD_MANAGE_KERNEL_EXIT";
+
+	if (check_env_var(var)) {
+		pthread_exit(NULL);
+	}
+}
+
+void __testpoint_thread_manage_consumer(void)
+{
+	const char *var = "LTTNG_THREAD_MANAGE_CONSUMER_EXIT";
+
+	if (check_env_var(var)) {
+		pthread_exit(NULL);
+	}
+}
diff --git a/tests/tools/health/health_thread_exit b/tests/tools/health/health_thread_exit
new file mode 100755
index 0000000..dab6b64
--- /dev/null
+++ b/tests/tools/health/health_thread_exit
@@ -0,0 +1,105 @@
+#!/bin/bash
+#
+# Copyright (C) - 2012 Christian Babeux <christian.babeux@efficios.com>
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License, version 2 only, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+TEST_DESC="Health check - Thread exit"
+
+CURDIR=$(dirname $0)/
+TESTDIR=$CURDIR/../..
+LTTNG_BIN="lttng"
+SESSION_NAME="health_thread_exit"
+EVENT_NAME="bogus"
+HEALTH_CHECK_BIN="health_check"
+SESSIOND_PRELOAD=".libs/libhealthexit.so"
+
+source $TESTDIR/utils.sh
+
+print_test_banner "$TEST_DESC"
+
+if [ ! -f "$SESSIOND_PRELOAD" ]; then
+	echo -e "libhealthexit.so not available for this test. Skipping."
+	exit 0
+fi
+
+function test_thread_exit
+{
+	test_thread_exit_name="$1"
+	test_thread_exit_code="$2"
+
+	echo ""
+	echo -e "=== Testing health failure with ${test_thread_exit_name}"
+
+	# Activate testpoints
+	export LTTNG_TESTPOINT_ENABLE=1
+
+	# Activate specific thread exit
+	export ${test_thread_exit_name}_EXIT=1
+
+	# Spawn sessiond with preload healthexit lib
+	export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
+	start_lttng_sessiond
+
+	# Cleanup some env. var.
+	unset LD_PRELOAD
+	unset ${test_thread_exit_name}_EXIT
+
+	# Check initial health status
+	$CURDIR/$HEALTH_CHECK_BIN &> /dev/null
+
+	echo -n "Validating thread ${test_thread_exit_name} failure... "
+
+	# Wait
+	sleep 25
+
+	# Check health status, exit code should indicate failure
+	$CURDIR/$HEALTH_CHECK_BIN &> /dev/null
+
+	health_check_exit_code=$?
+
+	if [ $health_check_exit_code -eq $test_thread_exit_code ]; then
+		print_ok
+		stop_lttng_sessiond
+	else
+		print_fail
+		echo -e "Health returned: $health_check_exit_code\n"
+
+		stop_lttng_sessiond
+		return 1
+	fi
+}
+
+THREAD=("LTTNG_THREAD_MANAGE_CLIENTS"
+	"LTTNG_THREAD_MANAGE_APPS"
+	"LTTNG_THREAD_REG_APPS"
+	"LTTNG_THREAD_MANAGE_KERNEL")
+
+# Exit code value to indicate specific thread failure
+EXIT_CODE=(1 2 4 8)
+
+THREAD_COUNT=${#THREAD[@]}
+i=0
+while [ "$i" -lt "$THREAD_COUNT" ]; do
+	test_thread_exit "${THREAD[$i]}" "${EXIT_CODE[$i]}"
+
+	if [ $? -eq 1 ]; then
+		exit 1
+	fi
+
+	let "i++"
+done
+
+# Special case manage consumer, need to spawn consumer via commands.
+#"LTTNG_THREAD_MANAGE_CONSUMER"
-- 
1.7.12.1

[PATCH v2 lttng-tools 4/5] Tests: Add health check thread stall test

[Christian Babeux]

This test trigger a "code stall" in a specified thread using the
testpoint mechanism. The testpoint behavior is implemented in
health_stall.c. The testpoint code stall a specific thread processing
by calling sleep(3).

The test select the thread to be stalled by enabling a specific
environment variable.

The test ensure the threads can be succesfully stalled and that the
health check feature is able to properly detect stalling in non-polling
cases.

Signed-off-by: Christian Babeux <christian.babeux@efficios.com>
---
 tests/tools/health/Makefile.am         |   6 +-
 tests/tools/health/health_stall.c      |  67 +++++++++++++++++
 tests/tools/health/health_thread_stall | 128 +++++++++++++++++++++++++++++++++
 3 files changed, 200 insertions(+), 1 deletion(-)
 create mode 100644 tests/tools/health/health_stall.c
 create mode 100755 tests/tools/health/health_thread_stall

diff --git a/tests/tools/health/Makefile.am b/tests/tools/health/Makefile.am
index 0a3f6c5..9fab582 100644
--- a/tests/tools/health/Makefile.am
+++ b/tests/tools/health/Makefile.am
@@ -10,12 +10,16 @@ endif
 
 UTILS=
 
-lib_LTLIBRARIES=libhealthexit.la
+lib_LTLIBRARIES=libhealthexit.la libhealthstall.la
 
 # Health thread exit ld_preloaded test lib
 libhealthexit_la_SOURCES=health_exit.c
 libhealthexit_la_LDFLAGS= -module
 
+# Health thread stall ld_preloaded test lib
+libhealthstall_la_SOURCES=health_stall.c
+libhealthstall_la_LDFLAGS= -module
+
 noinst_PROGRAMS = health_check
 
 health_check_SOURCES = health_check.c $(UTILS)
diff --git a/tests/tools/health/health_stall.c b/tests/tools/health/health_stall.c
new file mode 100644
index 0000000..9ce3e65
--- /dev/null
+++ b/tests/tools/health/health_stall.c
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2012 - Christian Babeux <christian.babeux@efficios.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License, version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#define STALL_TIME 60
+
+/*
+ * Check if the specified environment variable is set.
+ * Return 1 if set, otherwise 0.
+ */
+int check_env_var(const char *env)
+{
+	if (env) {
+		char *env_val = getenv(env);
+		if (env_val && (strncmp(env_val, "1", 1) == 0)) {
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+void __testpoint_thread_manage_clients_before_loop(void)
+{
+	const char *var = "LTTNG_THREAD_MANAGE_CLIENTS_STALL";
+
+	if (check_env_var(var)) {
+		sleep(STALL_TIME);
+	}
+}
+
+void __testpoint_thread_manage_kernel_before_loop(void)
+{
+	const char *var = "LTTNG_THREAD_MANAGE_KERNEL_STALL";
+
+	if (check_env_var(var)) {
+		sleep(STALL_TIME);
+	}
+}
+
+void __testpoint_thread_manage_apps_before_loop(void)
+{
+	const char *var = "LTTNG_THREAD_MANAGE_APPS_STALL";
+
+	if (check_env_var(var)) {
+		sleep(STALL_TIME);
+	}
+}
+
diff --git a/tests/tools/health/health_thread_stall b/tests/tools/health/health_thread_stall
new file mode 100755
index 0000000..d870895
--- /dev/null
+++ b/tests/tools/health/health_thread_stall
@@ -0,0 +1,128 @@
+#!/bin/bash
+#
+# Copyright (C) - 2012 Christian Babeux <christian.babeux@efficios.com>
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License, version 2 only, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+TEST_DESC="Health check - Thread stall"
+
+CURDIR=$(dirname $0)/
+TESTDIR=$CURDIR/../..
+LTTNG_BIN="lttng"
+SESSION_NAME="health_thread_stall"
+EVENT_NAME="bogus"
+HEALTH_CHECK_BIN="health_check"
+SESSIOND_PRELOAD=".libs/libhealthstall.so"
+
+source $TESTDIR/utils.sh
+
+print_test_banner "$TEST_DESC"
+
+if [ ! -f "$SESSIOND_PRELOAD" ]; then
+	echo -e "libhealthstall.so not available for this test. Skipping."
+	exit 0
+fi
+
+function test_thread_stall
+{
+	test_thread_stall_name="$1"
+	test_thread_exit_code="$2"
+
+	echo ""
+	echo -e "=== Testing health failure with ${test_thread_stall_name}"
+
+	# Activate testpoints
+	export LTTNG_TESTPOINT_ENABLE=1
+
+	# Activate specific thread exit
+	export ${test_thread_stall_name}_STALL=1
+
+	# Spawn sessiond with preload healthexit lib
+	export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
+	start_lttng_sessiond
+
+	# Cleanup some env. var.
+	unset LD_PRELOAD
+	unset ${test_thread_stall_name}_STALL
+
+	# Check initial health status
+	$CURDIR/$HEALTH_CHECK_BIN &> /dev/null
+
+	echo -n "Validating that ${test_thread_stall_name} is stalled... "
+
+	# Wait
+	sleep 25
+
+	# Check health status, exit code should indicate failure
+	$CURDIR/$HEALTH_CHECK_BIN &> /dev/null
+
+	health_check_exit_code=$?
+
+	if [ $health_check_exit_code -eq $test_thread_exit_code ]; then
+		print_ok
+	else
+		print_fail
+		echo -e "Health returned: $health_check_exit_code\n"
+
+		stop_lttng_sessiond
+		return 1
+	fi
+
+	echo -n "Validating that ${test_thread_stall_name} is no longer stalled... "
+
+	# Wait
+	sleep 40
+
+	# Check health status, exit code should now pass
+	$CURDIR/$HEALTH_CHECK_BIN &> /dev/null
+
+	health_check_exit_code=$?
+
+	if [ $health_check_exit_code -eq 0 ]; then
+		print_ok
+		stop_lttng_sessiond
+	else
+		print_fail
+		echo -e "Health returned: $health_check_exit_code\n"
+		stop_lttng_sessiond
+		return 1
+	fi
+
+
+}
+
+THREAD=("LTTNG_THREAD_MANAGE_CLIENTS"
+	"LTTNG_THREAD_MANAGE_APPS"
+# This thread is a little bit tricky to stall,
+# need to send some commands and setup an app.
+#	"LTTNG_THREAD_REG_APPS"
+	"LTTNG_THREAD_MANAGE_KERNEL")
+
+# Exit code value to indicate specific thread failure
+EXIT_CODE=(1
+	   2
+#	   4
+	   8)
+
+THREAD_COUNT=${#THREAD[@]}
+i=0
+while [ "$i" -lt "$THREAD_COUNT" ]; do
+	test_thread_stall "${THREAD[$i]}" "${EXIT_CODE[$i]}"
+
+	if [ $? -eq 1 ]; then
+		exit 1
+	fi
+
+	let "i++"
+done
-- 
1.7.12.1

[PATCH v2 lttng-tools 1/5] Add testpoints in lttng-sessiond to instrument every threads

[Christian Babeux]

This commit adds 8 new testpoints in the lttng-sessiond binary.
These testpoints rely on the testpoints infrastructure introduced
recently.

Testpoints:

thread_manage_clients
thread_manage_clients_before_loop
thread_registration_apps
thread_manage_apps
thread_manage_apps_before_loop
thread_manage_kernel
thread_manage_kernel_before_loop
thread_manage_consumer

The thread_<thread_name> testpoints are placed directly at the thread
start and they can be used to trigger failure in <thread_name>.

The thread_<thread_name>_before_loop testpoints are placed
directly before the main processing loop of the thread and thus can be
used to stall the processing of the thread.

Signed-off-by: Christian Babeux <christian.babeux@efficios.com>
---
 src/bin/lttng-sessiond/Makefile.am |  3 ++-
 src/bin/lttng-sessiond/main.c      | 28 ++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/src/bin/lttng-sessiond/Makefile.am b/src/bin/lttng-sessiond/Makefile.am
index 73be023..733818e 100644
--- a/src/bin/lttng-sessiond/Makefile.am
+++ b/src/bin/lttng-sessiond/Makefile.am
@@ -38,7 +38,8 @@ lttng_sessiond_LDADD = -lrt -lurcu-common -lurcu \
 		$(top_builddir)/src/common/hashtable/libhashtable.la \
 		$(top_builddir)/src/common/libcommon.la \
 		$(top_builddir)/src/common/compat/libcompat.la \
-		$(top_builddir)/src/common/relayd/librelayd.la
+		$(top_builddir)/src/common/relayd/librelayd.la \
+		$(top_builddir)/src/common/testpoint/libtestpoint.la
 
 if HAVE_LIBLTTNG_UST_CTL
 lttng_sessiond_LDADD += -llttng-ust-ctl
diff --git a/src/bin/lttng-sessiond/main.c b/src/bin/lttng-sessiond/main.c
index 730ac65..6e93cf0 100644
--- a/src/bin/lttng-sessiond/main.c
+++ b/src/bin/lttng-sessiond/main.c
@@ -45,6 +45,7 @@
 #include <common/futex.h>
 #include <common/relayd/relayd.h>
 #include <common/utils.h>
+#include <common/testpoint/testpoint.h>
 
 #include "lttng-sessiond.h"
 #include "channel.h"
@@ -65,6 +66,16 @@
 
 #define CONSUMERD_FILE	"lttng-consumerd"
 
+/* Testpoints, internal use only */
+TESTPOINT_DECL(thread_manage_clients);
+TESTPOINT_DECL(thread_manage_clients_before_loop);
+TESTPOINT_DECL(thread_registration_apps);
+TESTPOINT_DECL(thread_manage_apps);
+TESTPOINT_DECL(thread_manage_apps_before_loop);
+TESTPOINT_DECL(thread_manage_kernel);
+TESTPOINT_DECL(thread_manage_kernel_before_loop);
+TESTPOINT_DECL(thread_manage_consumer);
+
 /* Const values */
 const char default_home_dir[] = DEFAULT_HOME_DIR;
 const char default_tracing_group[] = DEFAULT_TRACING_GROUP;
@@ -680,8 +691,12 @@ static void *thread_manage_kernel(void *data)
 
 	DBG("Thread manage kernel started");
 
+	testpoint(thread_manage_kernel);
+
 	health_code_update(&health_thread_kernel);
 
+	testpoint(thread_manage_kernel_before_loop);
+
 	ret = create_thread_poll_set(&events, 2);
 	if (ret < 0) {
 		goto error_poll_create;
@@ -829,6 +844,9 @@ static void *thread_manage_consumer(void *data)
 	/* Inifinite blocking call, waiting for transmission */
 restart:
 	health_poll_update(&consumer_data->health);
+
+	testpoint(thread_manage_consumer);
+
 	ret = lttng_poll_wait(&events, -1);
 	health_poll_update(&consumer_data->health);
 	if (ret < 0) {
@@ -1026,6 +1044,8 @@ static void *thread_manage_apps(void *data)
 
 	DBG("[thread] Manage application started");
 
+	testpoint(thread_manage_apps);
+
 	rcu_register_thread();
 	rcu_thread_online();
 
@@ -1041,6 +1061,8 @@ static void *thread_manage_apps(void *data)
 		goto error;
 	}
 
+	testpoint(thread_manage_apps_before_loop);
+
 	health_code_update(&health_thread_app_manage);
 
 	while (1) {
@@ -1264,6 +1286,8 @@ static void *thread_registration_apps(void *data)
 
 	DBG("[thread] Manage application registration started");
 
+	testpoint(thread_registration_apps);
+
 	ret = lttcomm_listen_unix_sock(apps_sock);
 	if (ret < 0) {
 		goto error_listen;
@@ -2912,6 +2936,8 @@ static void *thread_manage_clients(void *data)
 
 	DBG("[thread] Manage client started");
 
+	testpoint(thread_manage_clients);
+
 	rcu_register_thread();
 
 	health_code_update(&health_thread_cmd);
@@ -2943,6 +2969,8 @@ static void *thread_manage_clients(void *data)
 		kill(ppid, SIGUSR1);
 	}
 
+	testpoint(thread_manage_clients_before_loop);
+
 	health_code_update(&health_thread_cmd);
 
 	while (1) {
-- 
1.7.12.1

[PATCH v2 lttng-tools 2/5] Tests: Add a health check utility program

[Christian Babeux]

The health_check program is a simple utility to query the health
status of the different threads of the sessiond.

Sample output:

> ./health_check
Health check cmd: 0
Health check app. manage: 0
Health check app. registration: 0
Health check kernel: 0
Health check consumer: 0

The return code is encoded to indicate which thread has failed.

Signed-off-by: Christian Babeux <christian.babeux@efficios.com>
---
 tests/tools/health/Makefile.am    | 20 +++++++++++
 tests/tools/health/health_check.c | 73 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 tests/tools/health/Makefile.am
 create mode 100644 tests/tools/health/health_check.c

diff --git a/tests/tools/health/Makefile.am b/tests/tools/health/Makefile.am
new file mode 100644
index 0000000..09573db
--- /dev/null
+++ b/tests/tools/health/Makefile.am
@@ -0,0 +1,20 @@
+AM_CFLAGS = -I. -O2 -g -I../../../include
+AM_LDFLAGS =
+
+if LTTNG_TOOLS_BUILD_WITH_LIBDL
+AM_LDFLAGS += -ldl
+endif
+if LTTNG_TOOLS_BUILD_WITH_LIBC_DL
+AM_LDFLAGS += -lc
+endif
+
+UTILS=
+
+noinst_PROGRAMS = health_check
+
+health_check_SOURCES = health_check.c $(UTILS)
+health_check_LDADD = $(top_builddir)/src/lib/lttng-ctl/liblttng-ctl.la \
+		     $(top_builddir)/src/common/libcommon.la
+
+noinst_SCRIPTS =
+EXTRA_DIST =
diff --git a/tests/tools/health/health_check.c b/tests/tools/health/health_check.c
new file mode 100644
index 0000000..3eef110
--- /dev/null
+++ b/tests/tools/health/health_check.c
@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2012 - Christian Babeux <christian.babeux@efficios.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License, version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <stdio.h>
+
+#include "lttng/lttng.h"
+
+#define HEALTH_CMD_FAIL     (1 << 0)
+#define HEALTH_APP_MNG_FAIL (1 << 1)
+#define HEALTH_APP_REG_FAIL (1 << 2)
+#define HEALTH_KERNEL_FAIL  (1 << 3)
+#define HEALTH_CSMR_FAIL    (1 << 4)
+
+int main(int argc, char *argv[])
+{
+	int health = -1;
+	int status = 0;
+
+	/* Command thread */
+	health = lttng_health_check(LTTNG_HEALTH_CMD);
+	printf("Health check cmd: %d\n", health);
+
+	if (health) {
+		status |= HEALTH_CMD_FAIL;
+	}
+
+	/* App manage thread */
+	health = lttng_health_check(LTTNG_HEALTH_APP_MANAGE);
+	printf("Health check app. manage: %d\n", health);
+
+	if (health) {
+		status |= HEALTH_APP_MNG_FAIL;
+	}
+	/* App registration thread */
+	health = lttng_health_check(LTTNG_HEALTH_APP_REG);
+	printf("Health check app. registration: %d\n", health);
+
+	if (health) {
+		status |= HEALTH_APP_REG_FAIL;
+	}
+
+	/* Kernel thread */
+	health = lttng_health_check(LTTNG_HEALTH_KERNEL);
+	printf("Health check kernel: %d\n", health);
+
+	if (health) {
+		status |= HEALTH_KERNEL_FAIL;
+	}
+
+	/* Consumer thread */
+	health = lttng_health_check(LTTNG_HEALTH_CONSUMER);
+	printf("Health check consumer: %d\n", health);
+
+	if (health) {
+		status |= HEALTH_CSMR_FAIL;
+	}
+
+	return status;
+}
-- 
1.7.12.1

[PATCH v3 lttng-tools 1/5] Add testpoints in lttng-sessiond to instrument every threads

[Christian Babeux]

This commit adds 8 new testpoints in the lttng-sessiond binary.
These testpoints rely on the testpoints infrastructure introduced
recently.

Testpoints:

thread_manage_clients
thread_manage_clients_before_loop
thread_registration_apps
thread_manage_apps
thread_manage_apps_before_loop
thread_manage_kernel
thread_manage_kernel_before_loop
thread_manage_consumer

The thread_<thread_name> testpoints are placed directly at the thread
start and they can be used to trigger failure in <thread_name>.

The thread_<thread_name>_before_loop testpoints are placed
directly before the main processing loop of the thread and thus can be
used to stall the processing of the thread.

Signed-off-by: Christian Babeux <christian.babeux@efficios.com>
---
 src/bin/lttng-sessiond/Makefile.am |  6 ++++--
 src/bin/lttng-sessiond/main.c      | 18 ++++++++++++++++++
 src/bin/lttng-sessiond/testpoint.h | 29 +++++++++++++++++++++++++++++
 3 files changed, 51 insertions(+), 2 deletions(-)
 create mode 100644 src/bin/lttng-sessiond/testpoint.h

diff --git a/src/bin/lttng-sessiond/Makefile.am b/src/bin/lttng-sessiond/Makefile.am
index 73be023..b13059d 100644
--- a/src/bin/lttng-sessiond/Makefile.am
+++ b/src/bin/lttng-sessiond/Makefile.am
@@ -21,7 +21,8 @@ lttng_sessiond_SOURCES = utils.c utils.h \
                        kernel-consumer.c kernel-consumer.h \
                        consumer.h filter.c filter.h \
                        health.c health.h \
-                       cmd.c cmd.h
+                       cmd.c cmd.h \
+                       testpoint.h
 
 if HAVE_LIBLTTNG_UST_CTL
 lttng_sessiond_SOURCES += trace-ust.c ust-app.c ust-consumer.c ust-consumer.h
@@ -38,7 +39,8 @@ lttng_sessiond_LDADD = -lrt -lurcu-common -lurcu \
 		$(top_builddir)/src/common/hashtable/libhashtable.la \
 		$(top_builddir)/src/common/libcommon.la \
 		$(top_builddir)/src/common/compat/libcompat.la \
-		$(top_builddir)/src/common/relayd/librelayd.la
+		$(top_builddir)/src/common/relayd/librelayd.la \
+		$(top_builddir)/src/common/testpoint/libtestpoint.la
 
 if HAVE_LIBLTTNG_UST_CTL
 lttng_sessiond_LDADD += -llttng-ust-ctl
diff --git a/src/bin/lttng-sessiond/main.c b/src/bin/lttng-sessiond/main.c
index 730ac65..2b78141 100644
--- a/src/bin/lttng-sessiond/main.c
+++ b/src/bin/lttng-sessiond/main.c
@@ -62,6 +62,7 @@
 #include "fd-limit.h"
 #include "filter.h"
 #include "health.h"
+#include "testpoint.h"
 
 #define CONSUMERD_FILE	"lttng-consumerd"
 
@@ -680,8 +681,12 @@ static void *thread_manage_kernel(void *data)
 
 	DBG("Thread manage kernel started");
 
+	testpoint(thread_manage_kernel);
+
 	health_code_update(&health_thread_kernel);
 
+	testpoint(thread_manage_kernel_before_loop);
+
 	ret = create_thread_poll_set(&events, 2);
 	if (ret < 0) {
 		goto error_poll_create;
@@ -829,6 +834,9 @@ static void *thread_manage_consumer(void *data)
 	/* Inifinite blocking call, waiting for transmission */
 restart:
 	health_poll_update(&consumer_data->health);
+
+	testpoint(thread_manage_consumer);
+
 	ret = lttng_poll_wait(&events, -1);
 	health_poll_update(&consumer_data->health);
 	if (ret < 0) {
@@ -1026,6 +1034,8 @@ static void *thread_manage_apps(void *data)
 
 	DBG("[thread] Manage application started");
 
+	testpoint(thread_manage_apps);
+
 	rcu_register_thread();
 	rcu_thread_online();
 
@@ -1041,6 +1051,8 @@ static void *thread_manage_apps(void *data)
 		goto error;
 	}
 
+	testpoint(thread_manage_apps_before_loop);
+
 	health_code_update(&health_thread_app_manage);
 
 	while (1) {
@@ -1264,6 +1276,8 @@ static void *thread_registration_apps(void *data)
 
 	DBG("[thread] Manage application registration started");
 
+	testpoint(thread_registration_apps);
+
 	ret = lttcomm_listen_unix_sock(apps_sock);
 	if (ret < 0) {
 		goto error_listen;
@@ -2912,6 +2926,8 @@ static void *thread_manage_clients(void *data)
 
 	DBG("[thread] Manage client started");
 
+	testpoint(thread_manage_clients);
+
 	rcu_register_thread();
 
 	health_code_update(&health_thread_cmd);
@@ -2943,6 +2959,8 @@ static void *thread_manage_clients(void *data)
 		kill(ppid, SIGUSR1);
 	}
 
+	testpoint(thread_manage_clients_before_loop);
+
 	health_code_update(&health_thread_cmd);
 
 	while (1) {
diff --git a/src/bin/lttng-sessiond/testpoint.h b/src/bin/lttng-sessiond/testpoint.h
new file mode 100644
index 0000000..5116548
--- /dev/null
+++ b/src/bin/lttng-sessiond/testpoint.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2012 - Christian Babeux <christian.babeux@efficios.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <common/testpoint/testpoint.h>
+
+/* Testpoints, internal use only */
+TESTPOINT_DECL(thread_manage_clients);
+TESTPOINT_DECL(thread_manage_clients_before_loop);
+TESTPOINT_DECL(thread_registration_apps);
+TESTPOINT_DECL(thread_manage_apps);
+TESTPOINT_DECL(thread_manage_apps_before_loop);
+TESTPOINT_DECL(thread_manage_kernel);
+TESTPOINT_DECL(thread_manage_kernel_before_loop);
+TESTPOINT_DECL(thread_manage_consumer);
+
-- 
1.7.12.1