[PATCH v1 0/3] Keep-alive enhancements

[PATCH v1 0/3] Keep-alive enhancements

[Remy Horton]

This patchset adds enhancements to the keepalive core monitoring and
reporting sub-system. The first is support for idled (sleeping and
frequency-stepped) CPU cores, and the second is support for applications
to be notified of active as well as faulted cores. The latter is to allow
core state to be relayed to external (secondary) processes, which is
demonstrated by changes to the l2fed-keepalive example.

Remy Horton (3):
  eal: add new keepalive state & callback hook
  examples/l2fwd-keepalive: add IPC liveness reporting
  doc: add keepalive enhancement documentation

 doc/guides/rel_notes/release_16_07.rst          |   5 +
 examples/Makefile                               |   1 +
 examples/l2fwd-keepalive/Makefile               |   4 +-
 examples/l2fwd-keepalive/ka-agent/Makefile      |  51 ++++++++++
 examples/l2fwd-keepalive/ka-agent/main.c        | 128 +++++++++++++++++++++++
 examples/l2fwd-keepalive/main.c                 |  22 +++-
 examples/l2fwd-keepalive/shm.c                  | 130 ++++++++++++++++++++++++
 examples/l2fwd-keepalive/shm.h                  | 102 +++++++++++++++++++
 lib/librte_eal/bsdapp/eal/rte_eal_version.map   |   7 ++
 lib/librte_eal/common/include/rte_keepalive.h   |  40 ++++++++
 lib/librte_eal/common/rte_keepalive.c           |  35 ++++++-
 lib/librte_eal/linuxapp/eal/rte_eal_version.map |   7 ++
 12 files changed, 526 insertions(+), 6 deletions(-)
 create mode 100644 examples/l2fwd-keepalive/ka-agent/Makefile
 create mode 100644 examples/l2fwd-keepalive/ka-agent/main.c
 create mode 100644 examples/l2fwd-keepalive/shm.c
 create mode 100644 examples/l2fwd-keepalive/shm.h

-- 
2.5.5

[PATCH v1 1/3] eal: add new keepalive state & callback hook

[Remy Horton]

Signed-off-by: Remy Horton <remy.horton@intel.com>
---
 lib/librte_eal/bsdapp/eal/rte_eal_version.map   |  7 +++++
 lib/librte_eal/common/include/rte_keepalive.h   | 40 +++++++++++++++++++++++++
 lib/librte_eal/common/rte_keepalive.c           | 35 ++++++++++++++++++++--
 lib/librte_eal/linuxapp/eal/rte_eal_version.map |  7 +++++
 4 files changed, 87 insertions(+), 2 deletions(-)

diff --git a/lib/librte_eal/bsdapp/eal/rte_eal_version.map b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
index 58c2951..9a33441 100644
--- a/lib/librte_eal/bsdapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
@@ -151,3 +151,10 @@ DPDK_16.04 {
 	rte_eal_primary_proc_alive;
 
 } DPDK_2.2;
+
+DPDK_16.7 {
+	global:
+
+	rte_keepalive_register_alive_callback;
+
+} DPDK_16.04;
diff --git a/lib/librte_eal/common/include/rte_keepalive.h b/lib/librte_eal/common/include/rte_keepalive.h
index 10dac2e..3159730 100644
--- a/lib/librte_eal/common/include/rte_keepalive.h
+++ b/lib/librte_eal/common/include/rte_keepalive.h
@@ -59,6 +59,16 @@ typedef void (*rte_keepalive_failure_callback_t)(
 	const int id_core);
 
 /**
+ * Keepalive 'alive' callback.
+ *
+ *  Receives a data pointer passed to rte_keepalive_register_alive_callback()
+ *  and the id of the failed core.
+ */
+typedef void (*rte_keepalive_alive_callback_t)(
+	void *data,
+	const int id_core);
+
+/**
  * Keepalive state structure.
  * @internal
  */
@@ -105,4 +115,34 @@ void rte_keepalive_register_core(struct rte_keepalive *keepcfg,
 void
 rte_keepalive_mark_alive(struct rte_keepalive *keepcfg);
 
+/**
+ * Per-core sleep-time indication.
+ * @param *keepcfg
+ *   Keepalive structure pointer
+ *
+ * This function needs to be called from within the main process loop of
+ * the LCore going to sleep.
+ */
+void
+rte_keepalive_mark_sleep(struct rte_keepalive *keepcfg);
+
+/**
+ * Registers a 'live core' callback.
+ *
+ * The complement of the 'dead core' callback. This is called when a
+ * core is known to be alive, and is intended for cases when an app
+ * needs to know 'liveness' beyond just knowing when a core has died.
+ *
+ * @param *keepcfg
+ *   Keepalive structure pointer
+ * @param callback
+ *   Function called upon detection of a dead core.
+ * @param data
+ *   Data pointer to be passed to function callback.
+ */
+void
+rte_keepalive_register_alive_callback(struct rte_keepalive *keepcfg,
+	rte_keepalive_alive_callback_t callback,
+	void *data);
+
 #endif /* _KEEPALIVE_H_ */
diff --git a/lib/librte_eal/common/rte_keepalive.c b/lib/librte_eal/common/rte_keepalive.c
index 23363ec..7af3558 100644
--- a/lib/librte_eal/common/rte_keepalive.c
+++ b/lib/librte_eal/common/rte_keepalive.c
@@ -46,7 +46,8 @@ struct rte_keepalive {
 		ALIVE = 1,
 		MISSING = 0,
 		DEAD = 2,
-		GONE = 3
+		GONE = 3,
+		SLEEP = 4
 	} __rte_cache_aligned state_flags[RTE_KEEPALIVE_MAXCORES];
 
 	/** Last-seen-alive timestamps */
@@ -68,6 +69,15 @@ struct rte_keepalive {
 	void *callback_data;
 	uint64_t tsc_initial;
 	uint64_t tsc_mhz;
+
+	/** Live core handler. */
+	rte_keepalive_failure_callback_t alive_callback;
+
+	/**
+	 * Live core handler app data.
+	 * Pointer is passed to live core handler.
+	 */
+	void *alive_callback_data;
 };
 
 static void
@@ -95,6 +105,11 @@ rte_keepalive_dispatch_pings(__rte_unused void *ptr_timer,
 		case ALIVE: /* Alive */
 			keepcfg->state_flags[idx_core] = MISSING;
 			keepcfg->last_alive[idx_core] = rte_rdtsc();
+			if (keepcfg->alive_callback)
+				keepcfg->alive_callback(
+					keepcfg->alive_callback_data,
+					idx_core
+					);
 			break;
 		case MISSING: /* MIA */
 			print_trace("Core MIA. ", keepcfg, idx_core);
@@ -111,6 +126,8 @@ rte_keepalive_dispatch_pings(__rte_unused void *ptr_timer,
 			break;
 		case GONE: /* Buried */
 			break;
+		case SLEEP: /* Idled core */
+			break;
 		}
 	}
 }
@@ -133,11 +150,19 @@ rte_keepalive_create(rte_keepalive_failure_callback_t callback,
 	return keepcfg;
 }
 
+void rte_keepalive_register_alive_callback(struct rte_keepalive *keepcfg,
+	rte_keepalive_failure_callback_t callback,
+	void *data)
+{
+	keepcfg->alive_callback = callback;
+	keepcfg->alive_callback_data = data;
+}
+
 void
 rte_keepalive_register_core(struct rte_keepalive *keepcfg, const int id_core)
 {
 	if (id_core < RTE_KEEPALIVE_MAXCORES) {
-		keepcfg->active_cores[id_core] = 1;
+		keepcfg->active_cores[id_core] = ALIVE;
 		keepcfg->last_alive[id_core] = rte_rdtsc();
 	}
 }
@@ -147,3 +172,9 @@ rte_keepalive_mark_alive(struct rte_keepalive *keepcfg)
 {
 	keepcfg->state_flags[rte_lcore_id()] = ALIVE;
 }
+
+void
+rte_keepalive_mark_sleep(struct rte_keepalive *keepcfg)
+{
+	keepcfg->state_flags[rte_lcore_id()] = SLEEP;
+}
diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
index 12503ef..862bc92 100644
--- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
@@ -154,3 +154,10 @@ DPDK_16.04 {
 	rte_eal_primary_proc_alive;
 
 } DPDK_2.2;
+
+DPDK_16.7 {
+	global:
+
+	rte_keepalive_register_alive_callback;
+
+} DPDK_16.04;
-- 
2.5.5

[PATCH v1 2/3] examples/l2fwd-keepalive: add IPC liveness reporting

[Remy Horton]

Signed-off-by: Remy Horton <remy.horton@intel.com>
---
 examples/Makefile                          |   1 +
 examples/l2fwd-keepalive/Makefile          |   4 +-
 examples/l2fwd-keepalive/ka-agent/Makefile |  51 +++++++++++
 examples/l2fwd-keepalive/ka-agent/main.c   | 128 ++++++++++++++++++++++++++++
 examples/l2fwd-keepalive/main.c            |  22 ++++-
 examples/l2fwd-keepalive/shm.c             | 130 +++++++++++++++++++++++++++++
 examples/l2fwd-keepalive/shm.h             | 102 ++++++++++++++++++++++
 7 files changed, 434 insertions(+), 4 deletions(-)
 create mode 100644 examples/l2fwd-keepalive/ka-agent/Makefile
 create mode 100644 examples/l2fwd-keepalive/ka-agent/main.c
 create mode 100644 examples/l2fwd-keepalive/shm.c
 create mode 100644 examples/l2fwd-keepalive/shm.h

diff --git a/examples/Makefile b/examples/Makefile
index b28b30e..bd688b9 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -64,6 +64,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += l2fwd-crypto
 DIRS-$(CONFIG_RTE_LIBRTE_IVSHMEM) += l2fwd-ivshmem
 DIRS-$(CONFIG_RTE_LIBRTE_JOBSTATS) += l2fwd-jobstats
 DIRS-y += l2fwd-keepalive
+DIRS-y += l2fwd-keepalive/ka-agent
 DIRS-$(CONFIG_RTE_LIBRTE_LPM) += l3fwd
 DIRS-$(CONFIG_RTE_LIBRTE_ACL) += l3fwd-acl
 ifeq ($(CONFIG_RTE_LIBRTE_LPM),y)
diff --git a/examples/l2fwd-keepalive/Makefile b/examples/l2fwd-keepalive/Makefile
index 568edcb..3fcf513 100644
--- a/examples/l2fwd-keepalive/Makefile
+++ b/examples/l2fwd-keepalive/Makefile
@@ -1,6 +1,6 @@
 #   BSD LICENSE
 #
-#   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+#   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
 #   All rights reserved.
 #
 #   Redistribution and use in source and binary forms, with or without
@@ -42,7 +42,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
 APP = l2fwd-keepalive
 
 # all source are stored in SRCS-y
-SRCS-y := main.c
+SRCS-y := main.c shm.c
 
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
diff --git a/examples/l2fwd-keepalive/ka-agent/Makefile b/examples/l2fwd-keepalive/ka-agent/Makefile
new file mode 100644
index 0000000..4eaac76
--- /dev/null
+++ b/examples/l2fwd-keepalive/ka-agent/Makefile
@@ -0,0 +1,51 @@
+#   BSD LICENSE
+#
+#   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Intel Corporation nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overridden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# binary name
+APP = ka-agent
+
+# all source are stored in SRCS-y
+SRCS-y := main.c
+
+CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR)/../
+
+EXTRA_CFLAGS += -O3 -g -Wfatal-errors
+
+include $(RTE_SDK)/mk/rte.extapp.mk
diff --git a/examples/l2fwd-keepalive/ka-agent/main.c b/examples/l2fwd-keepalive/ka-agent/main.c
new file mode 100644
index 0000000..f05e3a5
--- /dev/null
+++ b/examples/l2fwd-keepalive/ka-agent/main.c
@@ -0,0 +1,128 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/wait.h>
+#include <sys/queue.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <time.h>
+
+#include <rte_keepalive.h>
+
+#include <shm.h>
+
+static struct rte_keepalive_shm *ka_shm_create(void)
+{
+	int fd = shm_open(RTE_KEEPALIVE_SHM_NAME, O_RDWR, 0666);
+	size_t size = sizeof(struct rte_keepalive_shm);
+	struct rte_keepalive_shm *shm;
+
+	if (fd < 0)
+		printf("Failed to open %s as SHM:%s\n",
+			RTE_KEEPALIVE_SHM_NAME,
+		strerror(errno));
+	else {
+		shm = (struct rte_keepalive_shm *) mmap(
+			0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+		close(fd);
+		if (shm == MAP_FAILED)
+			printf("Failed to mmap SHM:%s\n", strerror(errno));
+		else
+			return shm;
+	}
+
+	/* Reset to zero, as it was set to MAP_FAILED aka: (void *)-1 */
+	shm = 0;
+	return NULL;
+}
+
+int main(void)
+{
+	struct rte_keepalive_shm *shm = ka_shm_create();
+	struct timespec timeout = { .tv_nsec = 0 };
+	time_t start_time;
+	int idx_core;
+	int cnt_cores;
+
+	if (shm == NULL) {
+		printf("Unable to access shared core state\n");
+		return 1;
+	}
+	while (1) {
+		if (shm->magic == RTE_KEEPALIVE_SHM_MAGIC)
+			break;
+		printf("Shared KA memory not setup. Sleeping..\n");
+		sleep(5);
+	}
+	start_time = shm->time_of_init;
+	while (1) {
+		if (start_time != shm->time_of_init) {
+			printf("Signature mismatch.\n");
+			break;
+		}
+
+		timeout.tv_sec = time(NULL) + 2;
+		if (sem_timedwait(&shm->core_died, &timeout) == -1)
+			continue;
+
+		cnt_cores = 0;
+		for (idx_core = 0; idx_core < RTE_KEEPALIVE_MAXCORES;
+				idx_core++)
+			if (shm->core_state[idx_core] == 2)
+				cnt_cores++;
+		if (cnt_cores == 0) {
+			/* Can happen if core was restarted since Semaphore
+			 * was sent, due to agent being offline.
+			 */
+			printf("Warning: Empty dead core report\n");
+			continue;
+		}
+
+		printf("%i dead cores: ", cnt_cores);
+		for (idx_core = 0;
+				idx_core < RTE_KEEPALIVE_MAXCORES;
+				idx_core++)
+			if (shm->core_state[idx_core] == 2)
+				printf("%d, ", idx_core);
+		printf("\b\b\n");
+	}
+	if (munmap(shm, sizeof(struct rte_keepalive_shm)) != 0)
+		printf("Warning: munmap() failed\n");
+	return 0;
+}
diff --git a/examples/l2fwd-keepalive/main.c b/examples/l2fwd-keepalive/main.c
index 8da89aa..8ebf1fd 100644
--- a/examples/l2fwd-keepalive/main.c
+++ b/examples/l2fwd-keepalive/main.c
@@ -72,6 +72,8 @@
 #include <rte_timer.h>
 #include <rte_keepalive.h>
 
+#include "shm.h"
+
 #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1
 
 #define NB_MBUF   8192
@@ -523,17 +525,25 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask)
 }
 
 static void
-dead_core(__attribute__((unused)) void *ptr_data, const int id_core)
+dead_core(void *ptr_data, const int id_core)
 {
 	printf("Dead core %i - restarting..\n", id_core);
 	if (rte_eal_get_lcore_state(id_core) == FINISHED) {
 		rte_eal_wait_lcore(id_core);
 		rte_eal_remote_launch(l2fwd_launch_one_lcore, NULL, id_core);
+		rte_keepalive_shm_dead((struct rte_keepalive_shm *)ptr_data,
+			id_core);
 	} else {
 		printf("..false positive!\n");
 	}
 }
 
+static void
+alive_core(void *ptr_data, const int id_core)
+{
+	rte_keepalive_shm_alive((struct rte_keepalive_shm *)ptr_data, id_core);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -725,10 +735,18 @@ main(int argc, char **argv)
 	rte_timer_init(&stats_timer);
 
 	if (check_period > 0) {
+		struct rte_keepalive_shm *ka_shm;
+
+		ka_shm = rte_keepalive_shm_create();
+		if (ka_shm == NULL)
+			rte_exit(EXIT_FAILURE,
+				"rte_keepalive_shm_create() failed");
 		rte_global_keepalive_info =
-			rte_keepalive_create(&dead_core, NULL);
+			rte_keepalive_create(&dead_core, ka_shm);
 		if (rte_global_keepalive_info == NULL)
 			rte_exit(EXIT_FAILURE, "init_keep_alive() failed");
+		rte_keepalive_register_alive_callback(rte_global_keepalive_info,
+			alive_core, ka_shm);
 		rte_timer_init(&hb_timer);
 		if (rte_timer_reset(&hb_timer,
 				(check_period * rte_get_timer_hz()) / 1000,
diff --git a/examples/l2fwd-keepalive/shm.c b/examples/l2fwd-keepalive/shm.c
new file mode 100644
index 0000000..f94ddc2
--- /dev/null
+++ b/examples/l2fwd-keepalive/shm.c
@@ -0,0 +1,130 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <time.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_keepalive.h>
+
+#include "shm.h"
+
+struct rte_keepalive_shm *rte_keepalive_shm_create(void)
+{
+	int fd;
+	int idx_core;
+	struct rte_keepalive_shm *ka_shm;
+
+	/* If any existing object is not unlinked, it makes it all too easy
+	 * for clients to end up with stale shared memory blocks when
+	 * restarted. Unlinking makes sure subsequent shm_open by clients
+	 * will get the new block mapped below.
+	 */
+	if (shm_unlink(RTE_KEEPALIVE_SHM_NAME) == -1 && errno != ENOENT)
+		printf("Warning: Error unlinking stale %s (%s)\n",
+			RTE_KEEPALIVE_SHM_NAME, strerror(errno));
+
+	fd = shm_open(RTE_KEEPALIVE_SHM_NAME,
+		O_CREAT | O_TRUNC | O_RDWR, 0666);
+	if (fd < 0)
+		RTE_LOG(INFO, EAL,
+			"Failed to open %s as SHM (%s)\n",
+			RTE_KEEPALIVE_SHM_NAME,
+			strerror(errno));
+	else if (ftruncate(fd, sizeof(struct rte_keepalive_shm)) != 0)
+		RTE_LOG(INFO, EAL,
+			"Failed to resize SHM (%s)\n", strerror(errno));
+	else {
+		ka_shm = (struct rte_keepalive_shm *) mmap(
+			0, sizeof(struct rte_keepalive_shm),
+			PROT_READ | PROT_WRITE,	MAP_SHARED, fd, 0);
+		close(fd);
+		if (ka_shm == MAP_FAILED)
+			RTE_LOG(INFO, EAL,
+				"Failed to mmap SHM (%s)\n", strerror(errno));
+		else {
+			memset(ka_shm, 0, sizeof(struct rte_keepalive_shm));
+
+			/* Initialize the semaphores for IPC/SHM use */
+			if (sem_init(&ka_shm->core_died, 1, 0) != 0) {
+				RTE_LOG(INFO, EAL,
+					"Failed to setup SHM semaphore (%s)\n",
+					strerror(errno));
+				return NULL;
+			}
+
+			/* Set all cores to 'not present' */
+			for (idx_core = 0;
+				idx_core < RTE_KEEPALIVE_MAXCORES;
+				idx_core++)
+				ka_shm->core_state[idx_core] = -1;
+
+			/* Set magic number so agent knows setup
+			 * has finished.
+			 */
+			ka_shm->magic = RTE_KEEPALIVE_SHM_MAGIC;
+			ka_shm->time_of_init = time(NULL);
+
+			return ka_shm;
+		}
+	}
+return NULL;
+}
+
+void rte_keepalive_shm_alive(struct rte_keepalive_shm *shm, const int id_core)
+{
+	shm->core_state[id_core] = RTE_KEEPALIVE_SHM_ALIVE;
+}
+
+void rte_keepalive_shm_dead(struct rte_keepalive_shm *shm, const int id_core)
+{
+	int count;
+
+	shm->core_state[id_core] = RTE_KEEPALIVE_SHM_DEAD;
+
+	/* Limit number of times semaphore can be incremented, in case
+	 * listening agent is not active.
+	 */
+	if (sem_getvalue(&shm->core_died, &count) == -1) {
+		RTE_LOG(INFO, EAL, "Semaphore check failed(%s)\n",
+			strerror(errno));
+		return;
+	}
+	if (count > 1)
+		return;
+
+	if (sem_post(&shm->core_died) != 0)
+		RTE_LOG(INFO, EAL,
+			"Failed to increment semaphore (%s)\n",
+			strerror(errno));
+}
diff --git a/examples/l2fwd-keepalive/shm.h b/examples/l2fwd-keepalive/shm.h
new file mode 100644
index 0000000..f2a58cc
--- /dev/null
+++ b/examples/l2fwd-keepalive/shm.h
@@ -0,0 +1,102 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define RTE_KEEPALIVE_SHM_NAME "/dpdk_keepalive_shm_name"
+#define RTE_KEEPALIVE_SHM_MAGIC 0xff00aaf0
+
+#define RTE_KEEPALIVE_SHM_ALIVE 1
+#define RTE_KEEPALIVE_SHM_DEAD 2
+
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <semaphore.h>
+
+/**
+ * Keepalive SHM structure.
+ *
+ * The shared memory allocated by the primary is this size, and contains the
+ * information as contained within this struct. A secondary may open the SHM,
+ * and read the contents.
+ */
+struct rte_keepalive_shm {
+	/** Initialisation check. */
+	uint32_t magic;
+
+	/* Last state refresh time.
+	 *
+	 * Time keepalive started. Used to detect shutdown.
+	 */
+	uint32_t time_of_init;
+
+	/** IPC semaphore. Posted when a core dies */
+	sem_t core_died;
+
+	/**
+	 * Relayed status of each core.
+	 *
+	 * Each entry takes on one of the following values:
+	 *  -1 if core not monitored by keepalive
+	 *   1 if core alive (the normal good state)
+	 *   2 if core is dead.
+	 *   @note: State 0 (MISSING) is not relayed.
+	 */
+	int32_t core_state[RTE_KEEPALIVE_MAXCORES];
+};
+
+/**
+ * Create shared host memory keepalive object.
+ * @return
+ *  Pointer to SHM keepalive structure, or NULL on failure.
+ */
+struct rte_keepalive_shm *rte_keepalive_shm_create(void);
+
+/**
+ * Registers given core as 'alive'
+ * @param *shm
+ *  Pointer to SHM keepalive structure.
+ * @param id_core
+ *  Id of affected core
+ */
+void rte_keepalive_shm_alive(struct rte_keepalive_shm *shm, const int id_core);
+
+/**
+ * Registers given core as 'dead'
+ * @param *shm
+ *  Pointer to SHM keepalive structure.
+ * @param id_core
+ *  Id of affected core
+ */
+void rte_keepalive_shm_dead(struct rte_keepalive_shm *shm, const int id_core);
-- 
2.5.5

[PATCH v1 3/3] doc: add keepalive enhancement documentation

[Remy Horton]

Signed-off-by: Remy Horton <remy.horton@intel.com>
---
 doc/guides/rel_notes/release_16_07.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/guides/rel_notes/release_16_07.rst b/doc/guides/rel_notes/release_16_07.rst
index 83c841b..7309877 100644
--- a/doc/guides/rel_notes/release_16_07.rst
+++ b/doc/guides/rel_notes/release_16_07.rst
@@ -34,6 +34,11 @@ This section should contain new features added in this release. Sample format:
 
   Refer to the previous release notes for examples.
 
+* **Added keepalive enhancements.**
+
+   Adds support for reporting LCore liveness to secondary processes and
+   support for idled CPUs.
+
 
 Resolved Issues
 ---------------
-- 
2.5.5

FW: [PATCH v1 0/3] Keep-alive enhancements

[Liu, Shucong]

Shucong:
	The validation for patches has been done.

-----Original Message-----
From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Horton, Remy
Sent: Friday, April 29, 2016 1:41 PM
To: dev@dpdk.org
Subject: [dpdk-dev] [PATCH v1 0/3] Keep-alive enhancements

This patchset adds enhancements to the keepalive core monitoring and reporting sub-system. The first is support for idled (sleeping and
frequency-stepped) CPU cores, and the second is support for applications to be notified of active as well as faulted cores. The latter is to allow core state to be relayed to external (secondary) processes, which is demonstrated by changes to the l2fed-keepalive example.

Remy Horton (3):
  eal: add new keepalive state & callback hook
  examples/l2fwd-keepalive: add IPC liveness reporting
  doc: add keepalive enhancement documentation

 doc/guides/rel_notes/release_16_07.rst          |   5 +
 examples/Makefile                               |   1 +
 examples/l2fwd-keepalive/Makefile               |   4 +-
 examples/l2fwd-keepalive/ka-agent/Makefile      |  51 ++++++++++
 examples/l2fwd-keepalive/ka-agent/main.c        | 128 +++++++++++++++++++++++
 examples/l2fwd-keepalive/main.c                 |  22 +++-
 examples/l2fwd-keepalive/shm.c                  | 130 ++++++++++++++++++++++++
 examples/l2fwd-keepalive/shm.h                  | 102 +++++++++++++++++++
 lib/librte_eal/bsdapp/eal/rte_eal_version.map   |   7 ++
 lib/librte_eal/common/include/rte_keepalive.h   |  40 ++++++++
 lib/librte_eal/common/rte_keepalive.c           |  35 ++++++-
 lib/librte_eal/linuxapp/eal/rte_eal_version.map |   7 ++
 12 files changed, 526 insertions(+), 6 deletions(-)  create mode 100644 examples/l2fwd-keepalive/ka-agent/Makefile
 create mode 100644 examples/l2fwd-keepalive/ka-agent/main.c
 create mode 100644 examples/l2fwd-keepalive/shm.c  create mode 100644 examples/l2fwd-keepalive/shm.h

--
2.5.5

[PATCH v2 0/3] Keep-alive enhancements

[Remy Horton]

This patchset adds enhancements to the keepalive core monitoring and
reporting sub-system. The first is support for idled (sleeping and
frequency-stepped) CPU cores, and the second is support for applications
to be notified of active as well as faulted cores. The latter is to allow
core state to be relayed to external (secondary) processes, which is
demonstrated by changes to the l2fed-keepalive example.

--

v2 changes:
* Some date & typos fixups
* State enum made public and extended with new states
* Generalised 'alive' callback to all states
* Last-alive shows gone-to-sleep time for idle cores
* Removed some redundant sanity checks
* Last-alive time exposed to application
* #define'd semaphore timeout
* Agent checks for dead keepalive

Remy Horton (3):
  eal: add new keepalive states & callback hooks
  examples/l2fwd-keepalive: add IPC liveness reporting
  doc: add keepalive enhancement documentation

 doc/guides/rel_notes/release_16_07.rst          |   5 +
 examples/Makefile                               |   3 +-
 examples/l2fwd-keepalive/Makefile               |   4 +-
 examples/l2fwd-keepalive/ka-agent/Makefile      |  51 ++++++++
 examples/l2fwd-keepalive/ka-agent/main.c        | 150 ++++++++++++++++++++++++
 examples/l2fwd-keepalive/main.c                 |  22 +++-
 examples/l2fwd-keepalive/shm.c                  | 128 ++++++++++++++++++++
 examples/l2fwd-keepalive/shm.h                  |  89 ++++++++++++++
 lib/librte_eal/bsdapp/eal/rte_eal_version.map   |   7 ++
 lib/librte_eal/common/include/rte_keepalive.h   |  55 ++++++++-
 lib/librte_eal/common/rte_keepalive.c           |  48 ++++++--
 lib/librte_eal/linuxapp/eal/rte_eal_version.map |   7 ++
 12 files changed, 556 insertions(+), 13 deletions(-)
 create mode 100644 examples/l2fwd-keepalive/ka-agent/Makefile
 create mode 100644 examples/l2fwd-keepalive/ka-agent/main.c
 create mode 100644 examples/l2fwd-keepalive/shm.c
 create mode 100644 examples/l2fwd-keepalive/shm.h

-- 
2.5.5

[PATCH v2 1/3] eal: add new keepalive states & callback hooks

[Remy Horton]

Signed-off-by: Remy Horton <remy.horton@intel.com>
---
 examples/Makefile                               |  2 +-
 lib/librte_eal/bsdapp/eal/rte_eal_version.map   |  7 ++++
 lib/librte_eal/common/include/rte_keepalive.h   | 55 ++++++++++++++++++++++++-
 lib/librte_eal/common/rte_keepalive.c           | 48 +++++++++++++++++----
 lib/librte_eal/linuxapp/eal/rte_eal_version.map |  7 ++++
 5 files changed, 110 insertions(+), 9 deletions(-)

diff --git a/examples/Makefile b/examples/Makefile
index b28b30e..3bc635a 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,6 +1,6 @@
 #   BSD LICENSE
 #
-#   Copyright(c) 2014 6WIND S.A.
+#   Copyright(c) 2016 6WIND S.A.
 #
 #   Redistribution and use in source and binary forms, with or without
 #   modification, are permitted provided that the following conditions
diff --git a/lib/librte_eal/bsdapp/eal/rte_eal_version.map b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
index 58c2951..9a33441 100644
--- a/lib/librte_eal/bsdapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
@@ -151,3 +151,10 @@ DPDK_16.04 {
 	rte_eal_primary_proc_alive;
 
 } DPDK_2.2;
+
+DPDK_16.7 {
+	global:
+
+	rte_keepalive_register_alive_callback;
+
+} DPDK_16.04;
diff --git a/lib/librte_eal/common/include/rte_keepalive.h b/lib/librte_eal/common/include/rte_keepalive.h
index 10dac2e..e23dda2 100644
--- a/lib/librte_eal/common/include/rte_keepalive.h
+++ b/lib/librte_eal/common/include/rte_keepalive.h
@@ -1,7 +1,7 @@
 /*-
  *   BSD LICENSE
  *
- *   Copyright 2015 Intel Shannon Ltd. All rights reserved.
+ *   Copyright 2015-2016 Intel Shannon Ltd. All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -48,6 +48,16 @@
 #define RTE_KEEPALIVE_MAXCORES RTE_MAX_LCORE
 #endif
 
+enum rte_keepalive_state {
+	UNUSED = 0,
+	ALIVE = 1,
+	MISSING = 4,
+	DEAD = 2,
+	GONE = 3,
+	DOZING = 5,
+	SLEEP = 6
+};
+
 /**
  * Keepalive failure callback.
  *
@@ -59,6 +69,19 @@ typedef void (*rte_keepalive_failure_callback_t)(
 	const int id_core);
 
 /**
+ * Keepalive relay callback.
+ *
+ *  Receives a data pointer passed to rte_keepalive_register_relay_callback()
+ *  and the id of the core for which state is to be forwarded.
+ */
+typedef void (*rte_keepalive_relay_callback_t)(
+	void *data,
+	const int id_core,
+	enum rte_keepalive_state core_state,
+	uint64_t last_seen
+	);
+
+/**
  * Keepalive state structure.
  * @internal
  */
@@ -105,4 +128,34 @@ void rte_keepalive_register_core(struct rte_keepalive *keepcfg,
 void
 rte_keepalive_mark_alive(struct rte_keepalive *keepcfg);
 
+/**
+ * Per-core sleep-time indication.
+ * @param *keepcfg
+ *   Keepalive structure pointer
+ *
+ * This function needs to be called from within the main process loop of
+ * the LCore going to sleep.
+ */
+void
+rte_keepalive_mark_sleep(struct rte_keepalive *keepcfg);
+
+/**
+ * Registers a 'live core' callback.
+ *
+ * The complement of the 'dead core' callback. This is called when a
+ * core is known to be alive, and is intended for cases when an app
+ * needs to know 'liveness' beyond just knowing when a core has died.
+ *
+ * @param *keepcfg
+ *   Keepalive structure pointer
+ * @param callback
+ *   Function called upon detection of a dead core.
+ * @param data
+ *   Data pointer to be passed to function callback.
+ */
+void
+rte_keepalive_register_relay_callback(struct rte_keepalive *keepcfg,
+	rte_keepalive_relay_callback_t callback,
+	void *data);
+
 #endif /* _KEEPALIVE_H_ */
diff --git a/lib/librte_eal/common/rte_keepalive.c b/lib/librte_eal/common/rte_keepalive.c
index 23363ec..9cc890c 100644
--- a/lib/librte_eal/common/rte_keepalive.c
+++ b/lib/librte_eal/common/rte_keepalive.c
@@ -42,12 +42,8 @@
 
 struct rte_keepalive {
 	/** Core Liveness. */
-	enum rte_keepalive_state {
-		ALIVE = 1,
-		MISSING = 0,
-		DEAD = 2,
-		GONE = 3
-	} __rte_cache_aligned state_flags[RTE_KEEPALIVE_MAXCORES];
+	enum rte_keepalive_state __rte_cache_aligned state_flags[
+		RTE_KEEPALIVE_MAXCORES];
 
 	/** Last-seen-alive timestamps */
 	uint64_t last_alive[RTE_KEEPALIVE_MAXCORES];
@@ -68,6 +64,15 @@ struct rte_keepalive {
 	void *callback_data;
 	uint64_t tsc_initial;
 	uint64_t tsc_mhz;
+
+	/** Core state relay handler. */
+	rte_keepalive_relay_callback_t relay_callback;
+
+	/**
+	 * Core state relay handler app data.
+	 * Pointer is passed to live core handler.
+	 */
+	void *relay_callback_data;
 };
 
 static void
@@ -92,6 +97,8 @@ rte_keepalive_dispatch_pings(__rte_unused void *ptr_timer,
 			continue;
 
 		switch (keepcfg->state_flags[idx_core]) {
+		case UNUSED:
+			break;
 		case ALIVE: /* Alive */
 			keepcfg->state_flags[idx_core] = MISSING;
 			keepcfg->last_alive[idx_core] = rte_rdtsc();
@@ -111,7 +118,20 @@ rte_keepalive_dispatch_pings(__rte_unused void *ptr_timer,
 			break;
 		case GONE: /* Buried */
 			break;
+		case DOZING: /* Core going idle */
+			keepcfg->state_flags[idx_core] = SLEEP;
+			keepcfg->last_alive[idx_core] = rte_rdtsc();
+			break;
+		case SLEEP: /* Idled core */
+			break;
 		}
+		if (keepcfg->relay_callback)
+			keepcfg->relay_callback(
+				keepcfg->relay_callback_data,
+				idx_core,
+				keepcfg->state_flags[idx_core],
+				keepcfg->last_alive[idx_core]
+				);
 	}
 }
 
@@ -133,11 +153,19 @@ rte_keepalive_create(rte_keepalive_failure_callback_t callback,
 	return keepcfg;
 }
 
+void rte_keepalive_register_relay_callback(struct rte_keepalive *keepcfg,
+	rte_keepalive_relay_callback_t callback,
+	void *data)
+{
+	keepcfg->relay_callback = callback;
+	keepcfg->relay_callback_data = data;
+}
+
 void
 rte_keepalive_register_core(struct rte_keepalive *keepcfg, const int id_core)
 {
 	if (id_core < RTE_KEEPALIVE_MAXCORES) {
-		keepcfg->active_cores[id_core] = 1;
+		keepcfg->active_cores[id_core] = ALIVE;
 		keepcfg->last_alive[id_core] = rte_rdtsc();
 	}
 }
@@ -147,3 +175,9 @@ rte_keepalive_mark_alive(struct rte_keepalive *keepcfg)
 {
 	keepcfg->state_flags[rte_lcore_id()] = ALIVE;
 }
+
+void
+rte_keepalive_mark_sleep(struct rte_keepalive *keepcfg)
+{
+	keepcfg->state_flags[rte_lcore_id()] = DOZING;
+}
diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
index 12503ef..862bc92 100644
--- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
@@ -154,3 +154,10 @@ DPDK_16.04 {
 	rte_eal_primary_proc_alive;
 
 } DPDK_2.2;
+
+DPDK_16.7 {
+	global:
+
+	rte_keepalive_register_alive_callback;
+
+} DPDK_16.04;
-- 
2.5.5

[PATCH v2 2/3] examples/l2fwd-keepalive: add IPC liveness reporting

[Remy Horton]

Signed-off-by: Remy Horton <remy.horton@intel.com>
---
 examples/Makefile                          |   1 +
 examples/l2fwd-keepalive/Makefile          |   4 +-
 examples/l2fwd-keepalive/ka-agent/Makefile |  51 ++++++++++
 examples/l2fwd-keepalive/ka-agent/main.c   | 150 +++++++++++++++++++++++++++++
 examples/l2fwd-keepalive/main.c            |  22 ++++-
 examples/l2fwd-keepalive/shm.c             | 128 ++++++++++++++++++++++++
 examples/l2fwd-keepalive/shm.h             |  89 +++++++++++++++++
 7 files changed, 441 insertions(+), 4 deletions(-)
 create mode 100644 examples/l2fwd-keepalive/ka-agent/Makefile
 create mode 100644 examples/l2fwd-keepalive/ka-agent/main.c
 create mode 100644 examples/l2fwd-keepalive/shm.c
 create mode 100644 examples/l2fwd-keepalive/shm.h

diff --git a/examples/Makefile b/examples/Makefile
index 3bc635a..f650d3e 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -64,6 +64,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += l2fwd-crypto
 DIRS-$(CONFIG_RTE_LIBRTE_IVSHMEM) += l2fwd-ivshmem
 DIRS-$(CONFIG_RTE_LIBRTE_JOBSTATS) += l2fwd-jobstats
 DIRS-y += l2fwd-keepalive
+DIRS-y += l2fwd-keepalive/ka-agent
 DIRS-$(CONFIG_RTE_LIBRTE_LPM) += l3fwd
 DIRS-$(CONFIG_RTE_LIBRTE_ACL) += l3fwd-acl
 ifeq ($(CONFIG_RTE_LIBRTE_LPM),y)
diff --git a/examples/l2fwd-keepalive/Makefile b/examples/l2fwd-keepalive/Makefile
index 568edcb..3fcf513 100644
--- a/examples/l2fwd-keepalive/Makefile
+++ b/examples/l2fwd-keepalive/Makefile
@@ -1,6 +1,6 @@
 #   BSD LICENSE
 #
-#   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+#   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
 #   All rights reserved.
 #
 #   Redistribution and use in source and binary forms, with or without
@@ -42,7 +42,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
 APP = l2fwd-keepalive
 
 # all source are stored in SRCS-y
-SRCS-y := main.c
+SRCS-y := main.c shm.c
 
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
diff --git a/examples/l2fwd-keepalive/ka-agent/Makefile b/examples/l2fwd-keepalive/ka-agent/Makefile
new file mode 100644
index 0000000..4eaac76
--- /dev/null
+++ b/examples/l2fwd-keepalive/ka-agent/Makefile
@@ -0,0 +1,51 @@
+#   BSD LICENSE
+#
+#   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Intel Corporation nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overridden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# binary name
+APP = ka-agent
+
+# all source are stored in SRCS-y
+SRCS-y := main.c
+
+CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR)/../
+
+EXTRA_CFLAGS += -O3 -g -Wfatal-errors
+
+include $(RTE_SDK)/mk/rte.extapp.mk
diff --git a/examples/l2fwd-keepalive/ka-agent/main.c b/examples/l2fwd-keepalive/ka-agent/main.c
new file mode 100644
index 0000000..c8c7303
--- /dev/null
+++ b/examples/l2fwd-keepalive/ka-agent/main.c
@@ -0,0 +1,150 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/wait.h>
+#include <sys/queue.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <time.h>
+
+#include <rte_keepalive.h>
+
+#include <shm.h>
+
+#define MAX_TIMEOUTS 4
+#define SEM_TIMEOUT_SECS 2
+
+static struct rte_keepalive_shm *ka_shm_create(void)
+{
+	int fd = shm_open(RTE_KEEPALIVE_SHM_NAME, O_RDWR, 0666);
+	size_t size = sizeof(struct rte_keepalive_shm);
+	struct rte_keepalive_shm *shm;
+
+	if (fd < 0)
+		printf("Failed to open %s as SHM:%s\n",
+			RTE_KEEPALIVE_SHM_NAME,
+		strerror(errno));
+	else {
+		shm = (struct rte_keepalive_shm *) mmap(
+			0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+		close(fd);
+		if (shm == MAP_FAILED)
+			printf("Failed to mmap SHM:%s\n", strerror(errno));
+		else
+			return shm;
+	}
+
+	/* Reset to zero, as it was set to MAP_FAILED aka: (void *)-1 */
+	shm = 0;
+	return NULL;
+}
+
+int main(void)
+{
+	struct rte_keepalive_shm *shm = ka_shm_create();
+	struct timespec timeout = { .tv_nsec = 0 };
+	int idx_core;
+	int cnt_cores;
+	uint64_t last_seen_alive_time = 0;
+	uint64_t most_recent_alive_time;
+	int cnt_timeouts = 0;
+	int sem_errno;
+
+	if (shm == NULL) {
+		printf("Unable to access shared core state\n");
+		return 1;
+	}
+	while (1) {
+		most_recent_alive_time = 0;
+		for (idx_core = 0; idx_core < RTE_KEEPALIVE_MAXCORES;
+				idx_core++)
+			if (shm->core_last_seen_times[idx_core] >
+					most_recent_alive_time)
+				most_recent_alive_time =
+					shm->core_last_seen_times[idx_core];
+
+		timeout.tv_sec = time(NULL) + SEM_TIMEOUT_SECS;
+		if (sem_timedwait(&shm->core_died, &timeout) == -1) {
+			/* Assume no core death signals and no change in any
+			 * last-seen times is the keepalive monitor itself
+			 * failing.
+			 */
+			sem_errno = errno;
+			last_seen_alive_time = most_recent_alive_time;
+			if (sem_errno == ETIMEDOUT) {
+				if (last_seen_alive_time ==
+						most_recent_alive_time &&
+						cnt_timeouts++ >
+						MAX_TIMEOUTS) {
+					printf("No updates. Exiting..\n");
+					break;
+					}
+			} else
+				printf("sem_timedwait() error (%s)\n",
+					strerror(sem_errno));
+			continue;
+		}
+		cnt_timeouts = 0;
+
+		cnt_cores = 0;
+		for (idx_core = 0; idx_core < RTE_KEEPALIVE_MAXCORES;
+				idx_core++)
+			if (shm->core_state[idx_core] == DEAD)
+				cnt_cores++;
+		if (cnt_cores == 0) {
+			/* Can happen if core was restarted since Semaphore
+			 * was sent, due to agent being offline.
+			 */
+			printf("Warning: Empty dead core report\n");
+			continue;
+		}
+
+		printf("%i dead cores: ", cnt_cores);
+		for (idx_core = 0;
+				idx_core < RTE_KEEPALIVE_MAXCORES;
+				idx_core++)
+			if (shm->core_state[idx_core] == DEAD)
+				printf("%d, ", idx_core);
+		printf("\b\b\n");
+	}
+	if (munmap(shm, sizeof(struct rte_keepalive_shm)) != 0)
+		printf("Warning: munmap() failed\n");
+	return 0;
+}
diff --git a/examples/l2fwd-keepalive/main.c b/examples/l2fwd-keepalive/main.c
index 8da89aa..a69cbb9 100644
--- a/examples/l2fwd-keepalive/main.c
+++ b/examples/l2fwd-keepalive/main.c
@@ -72,6 +72,8 @@
 #include <rte_timer.h>
 #include <rte_keepalive.h>
 
+#include "shm.h"
+
 #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1
 
 #define NB_MBUF   8192
@@ -523,7 +525,7 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask)
 }
 
 static void
-dead_core(__attribute__((unused)) void *ptr_data, const int id_core)
+dead_core(__rte_unused void *ptr_data, const int id_core)
 {
 	printf("Dead core %i - restarting..\n", id_core);
 	if (rte_eal_get_lcore_state(id_core) == FINISHED) {
@@ -534,6 +536,14 @@ dead_core(__attribute__((unused)) void *ptr_data, const int id_core)
 	}
 }
 
+static void
+relay_core_state(void *ptr_data, const int id_core,
+	const enum rte_keepalive_state core_state, uint64_t last_alive)
+{
+	rte_keepalive_relayed_state((struct rte_keepalive_shm *)ptr_data,
+		id_core, core_state, last_alive);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -725,10 +735,18 @@ main(int argc, char **argv)
 	rte_timer_init(&stats_timer);
 
 	if (check_period > 0) {
+		struct rte_keepalive_shm *ka_shm;
+
+		ka_shm = rte_keepalive_shm_create();
+		if (ka_shm == NULL)
+			rte_exit(EXIT_FAILURE,
+				"rte_keepalive_shm_create() failed");
 		rte_global_keepalive_info =
-			rte_keepalive_create(&dead_core, NULL);
+			rte_keepalive_create(&dead_core, ka_shm);
 		if (rte_global_keepalive_info == NULL)
 			rte_exit(EXIT_FAILURE, "init_keep_alive() failed");
+		rte_keepalive_register_relay_callback(rte_global_keepalive_info,
+			relay_core_state, ka_shm);
 		rte_timer_init(&hb_timer);
 		if (rte_timer_reset(&hb_timer,
 				(check_period * rte_get_timer_hz()) / 1000,
diff --git a/examples/l2fwd-keepalive/shm.c b/examples/l2fwd-keepalive/shm.c
new file mode 100644
index 0000000..f66bd12
--- /dev/null
+++ b/examples/l2fwd-keepalive/shm.c
@@ -0,0 +1,128 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <time.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_keepalive.h>
+
+#include "shm.h"
+
+struct rte_keepalive_shm *rte_keepalive_shm_create(void)
+{
+	int fd;
+	int idx_core;
+	struct rte_keepalive_shm *ka_shm;
+
+	/* If any existing object is not unlinked, it makes it all too easy
+	 * for clients to end up with stale shared memory blocks when
+	 * restarted. Unlinking makes sure subsequent shm_open by clients
+	 * will get the new block mapped below.
+	 */
+	if (shm_unlink(RTE_KEEPALIVE_SHM_NAME) == -1 && errno != ENOENT)
+		printf("Warning: Error unlinking stale %s (%s)\n",
+			RTE_KEEPALIVE_SHM_NAME, strerror(errno));
+
+	fd = shm_open(RTE_KEEPALIVE_SHM_NAME,
+		O_CREAT | O_TRUNC | O_RDWR, 0666);
+	if (fd < 0)
+		RTE_LOG(INFO, EAL,
+			"Failed to open %s as SHM (%s)\n",
+			RTE_KEEPALIVE_SHM_NAME,
+			strerror(errno));
+	else if (ftruncate(fd, sizeof(struct rte_keepalive_shm)) != 0)
+		RTE_LOG(INFO, EAL,
+			"Failed to resize SHM (%s)\n", strerror(errno));
+	else {
+		ka_shm = (struct rte_keepalive_shm *) mmap(
+			0, sizeof(struct rte_keepalive_shm),
+			PROT_READ | PROT_WRITE,	MAP_SHARED, fd, 0);
+		close(fd);
+		if (ka_shm == MAP_FAILED)
+			RTE_LOG(INFO, EAL,
+				"Failed to mmap SHM (%s)\n", strerror(errno));
+		else {
+			memset(ka_shm, 0, sizeof(struct rte_keepalive_shm));
+
+			/* Initialize the semaphores for IPC/SHM use */
+			if (sem_init(&ka_shm->core_died, 1, 0) != 0) {
+				RTE_LOG(INFO, EAL,
+					"Failed to setup SHM semaphore (%s)\n",
+					strerror(errno));
+				return NULL;
+			}
+
+			/* Set all cores to 'not present' */
+			for (idx_core = 0;
+					idx_core < RTE_KEEPALIVE_MAXCORES;
+					idx_core++) {
+				ka_shm->core_state[idx_core] = UNUSED;
+				ka_shm->core_last_seen_times[idx_core] = 0;
+			}
+
+			return ka_shm;
+		}
+	}
+return NULL;
+}
+
+void rte_keepalive_relayed_state(struct rte_keepalive_shm *shm,
+	const int id_core, const enum rte_keepalive_state core_state,
+	__rte_unused uint64_t last_alive)
+{
+	int count;
+
+	shm->core_state[id_core] = core_state;
+	shm->core_last_seen_times[id_core] = last_alive;
+
+	if (core_state == RTE_KEEPALIVE_SHM_DEAD) {
+		/* Since core has died, also signal ka_agent.
+		 *
+		 * Limit number of times semaphore can be incremented, in case
+		 * ka_agent is not active.
+		 */
+		if (sem_getvalue(&shm->core_died, &count) == -1) {
+			RTE_LOG(INFO, EAL, "Semaphore check failed(%s)\n",
+				strerror(errno));
+			return;
+		}
+		if (count > 1)
+			return;
+
+		if (sem_post(&shm->core_died) != 0)
+			RTE_LOG(INFO, EAL,
+				"Failed to increment semaphore (%s)\n",
+				strerror(errno));
+	}
+}
diff --git a/examples/l2fwd-keepalive/shm.h b/examples/l2fwd-keepalive/shm.h
new file mode 100644
index 0000000..25e1b61
--- /dev/null
+++ b/examples/l2fwd-keepalive/shm.h
@@ -0,0 +1,89 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define RTE_KEEPALIVE_SHM_NAME "/dpdk_keepalive_shm_name"
+
+#define RTE_KEEPALIVE_SHM_ALIVE 1
+#define RTE_KEEPALIVE_SHM_DEAD 2
+
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <semaphore.h>
+#include <rte_keepalive.h>
+
+/**
+ * Keepalive SHM structure.
+ *
+ * The shared memory allocated by the primary is this size, and contains the
+ * information as contained within this struct. A secondary may open the SHM,
+ * and read the contents.
+ */
+struct rte_keepalive_shm {
+	/** IPC semaphore. Posted when a core dies */
+	sem_t core_died;
+
+	/**
+	 * Relayed status of each core.
+	 */
+	enum rte_keepalive_state core_state[RTE_KEEPALIVE_MAXCORES];
+
+	/**
+	 * Last-seen-alive timestamps for the cores
+	 */
+	uint64_t core_last_seen_times[RTE_KEEPALIVE_MAXCORES];
+};
+
+/**
+ * Create shared host memory keepalive object.
+ * @return
+ *  Pointer to SHM keepalive structure, or NULL on failure.
+ */
+struct rte_keepalive_shm *rte_keepalive_shm_create(void);
+
+/**
+ * Relays state for given core
+ * @param *shm
+ *  Pointer to SHM keepalive structure.
+ * @param id_core
+ *  Id of core
+ * @param core_state
+ *  State of core
+ * @param last_alive
+ *  Last seen timestamp for core
+ */
+void rte_keepalive_relayed_state(struct rte_keepalive_shm *shm,
+	const int id_core, const enum rte_keepalive_state core_state,
+	uint64_t last_alive);
-- 
2.5.5

[PATCH v2 3/3] doc: add keepalive enhancement documentation

[Remy Horton]

Signed-off-by: Remy Horton <remy.horton@intel.com>
---
 doc/guides/rel_notes/release_16_07.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/doc/guides/rel_notes/release_16_07.rst b/doc/guides/rel_notes/release_16_07.rst
index f6d543c..bc269b0 100644
--- a/doc/guides/rel_notes/release_16_07.rst
+++ b/doc/guides/rel_notes/release_16_07.rst
@@ -34,6 +34,11 @@ This section should contain new features added in this release. Sample format:
 
   Refer to the previous release notes for examples.
 
+* **Added keepalive enhancements.**
+
+   Adds support for reporting LCore liveness to secondary processes and
+   support for idled CPUs.
+
 
 Resolved Issues
 ---------------
-- 
2.5.5

Re: [PATCH v2 0/3] Keep-alive enhancements

[Tahhan, Maryam]

> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Remy Horton
> Sent: Wednesday, May 18, 2016 10:30 AM
> To: dev@dpdk.org; Mcnamara, John <john.mcnamara@intel.com>
> Subject: [dpdk-dev] [PATCH v2 0/3] Keep-alive enhancements
> 
> This patchset adds enhancements to the keepalive core monitoring
> and reporting sub-system. The first is support for idled (sleeping and
> frequency-stepped) CPU cores, and the second is support for
> applications to be notified of active as well as faulted cores. The latter
> is to allow core state to be relayed to external (secondary) processes,
> which is demonstrated by changes to the l2fed-keepalive example.
> 
> --

Acked-by: Maryam Tahhan <maryam.tahhan@intel.com>

Re: [PATCH v2 3/3] doc: add keepalive enhancement documentation

[Thomas Monjalon]

2016-05-18 10:30, Remy Horton:

There is no explanation and it is totally normal, because this patch
must be squashed with the code change.

> Signed-off-by: Remy Horton <remy.horton@intel.com>
> ---
>  doc/guides/rel_notes/release_16_07.rst | 5 +++++
>  1 file changed, 5 insertions(+)
> 
> diff --git a/doc/guides/rel_notes/release_16_07.rst b/doc/guides/rel_notes/release_16_07.rst
> index f6d543c..bc269b0 100644
> --- a/doc/guides/rel_notes/release_16_07.rst
> +++ b/doc/guides/rel_notes/release_16_07.rst
> @@ -34,6 +34,11 @@ This section should contain new features added in this release. Sample format:
>  
>    Refer to the previous release notes for examples.
>  
> +* **Added keepalive enhancements.**
> +
> +   Adds support for reporting LCore liveness to secondary processes and
> +   support for idled CPUs.
> +

Re: [PATCH v2 1/3] eal: add new keepalive states & callback hooks

[Thomas Monjalon]

2016-05-18 10:30, Remy Horton:

The explanations are missing.
Probably you should split this patch.

> Signed-off-by: Remy Horton <remy.horton@intel.com>
[...]
> +enum rte_keepalive_state {
> +	UNUSED = 0,
> +	ALIVE = 1,
> +	MISSING = 4,
> +	DEAD = 2,
> +	GONE = 3,
> +	DOZING = 5,
> +	SLEEP = 6
> +};

Please use RTE_ prefix.

[...]
>  /**
> + * Keepalive relay callback.
> + *
> + *  Receives a data pointer passed to rte_keepalive_register_relay_callback()
> + *  and the id of the core for which state is to be forwarded.
> + */

Please document each parameter.

> +typedef void (*rte_keepalive_relay_callback_t)(
> +	void *data,
> +	const int id_core,
> +	enum rte_keepalive_state core_state,
> +	uint64_t last_seen
> +	);

[...]
> +/**
> + * Per-core sleep-time indication.
> + * @param *keepcfg
> + *   Keepalive structure pointer
> + *
> + * This function needs to be called from within the main process loop of
> + * the LCore going to sleep.

Why? Please add more comments.

> + */

Re: [PATCH v2 3/3] doc: add keepalive enhancement documentation

[Remy Horton]

On 08/06/2016 10:52, Thomas Monjalon wrote:
> 2016-05-18 10:30, Remy Horton:
>
> There is no explanation and it is totally normal, because this patch
> must be squashed with the code change.
>
>> Signed-off-by: Remy Horton <remy.horton@intel.com>
>> ---
>>   doc/guides/rel_notes/release_16_07.rst | 5 +++++
>>   1 file changed, 5 insertions(+)

Done for v3. I tend to keep documentation changes in a seperate commit
as from my experience they cause the most problems when rebasing.

..Remy

[PATCH v3 0/3] Keep-alive enhancements

[Remy Horton]

This patchset adds enhancements to the keepalive core monitoring and
reporting sub-system. The first is support for idled (sleeping and
frequency-stepped) CPU cores, and the second is support for applications
to be notified of active as well as faulted cores. The latter is to allow
core state to be relayed to external (secondary) processes, which is
demonstrated by changes to the l2fed-keepalive example.

--

v3 changes
* Rebased to master
* Changed keepalive state enumerations to use RTE_ prefix
* Added missing parameter documentation
* Doc changes squashed

v2 changes:
* Some date & typos fixups
* State enum made public and extended with new states
* Generalised 'alive' callback to all states
* Last-alive shows gone-to-sleep time for idle cores
* Removed some redundant sanity checks
* Last-alive time exposed to application
* #define'd semaphore timeout
* Agent checks for dead keepalive

Remy Horton (3):
  eal: export keepalive state enumerations
  eal: add additional keepalive callbacks
  examples/l2fwd-keepalive: add IPC liveness reporting

 doc/guides/rel_notes/release_16_07.rst          |   6 +
 examples/Makefile                               |   3 +-
 examples/l2fwd-keepalive/Makefile               |   4 +-
 examples/l2fwd-keepalive/ka-agent/Makefile      |  48 ++++++++
 examples/l2fwd-keepalive/ka-agent/main.c        | 150 ++++++++++++++++++++++++
 examples/l2fwd-keepalive/main.c                 |  22 +++-
 examples/l2fwd-keepalive/shm.c                  | 128 ++++++++++++++++++++
 examples/l2fwd-keepalive/shm.h                  |  89 ++++++++++++++
 lib/librte_eal/bsdapp/eal/rte_eal_version.map   |   1 +
 lib/librte_eal/common/include/rte_keepalive.h   |  63 +++++++++-
 lib/librte_eal/common/rte_keepalive.c           |  64 +++++++---
 lib/librte_eal/linuxapp/eal/rte_eal_version.map |   1 +
 12 files changed, 558 insertions(+), 21 deletions(-)
 create mode 100644 examples/l2fwd-keepalive/ka-agent/Makefile
 create mode 100644 examples/l2fwd-keepalive/ka-agent/main.c
 create mode 100644 examples/l2fwd-keepalive/shm.c
 create mode 100644 examples/l2fwd-keepalive/shm.h

-- 
2.5.5

[PATCH v3 1/3] eal: export keepalive state enumerations

[Remy Horton]

Changes the keepalive state from an anonymous enum to a declared one
which is externally visible, so that keepalive enum values can be
used by applications.

Signed-off-by: Remy Horton <remy.horton@intel.com>
---
 lib/librte_eal/common/include/rte_keepalive.h | 12 +++++++++-
 lib/librte_eal/common/rte_keepalive.c         | 34 +++++++++++++++------------
 2 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/lib/librte_eal/common/include/rte_keepalive.h b/lib/librte_eal/common/include/rte_keepalive.h
index 10dac2e..2f856c3 100644
--- a/lib/librte_eal/common/include/rte_keepalive.h
+++ b/lib/librte_eal/common/include/rte_keepalive.h
@@ -1,7 +1,7 @@
 /*-
  *   BSD LICENSE
  *
- *   Copyright 2015 Intel Shannon Ltd. All rights reserved.
+ *   Copyright 2015-2016 Intel Shannon Ltd. All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -48,6 +48,16 @@
 #define RTE_KEEPALIVE_MAXCORES RTE_MAX_LCORE
 #endif
 
+enum rte_keepalive_state {
+	RTE_UNUSED = 0,
+	RTE_ALIVE = 1,
+	RTE_MISSING = 4,
+	RTE_DEAD = 2,
+	RTE_GONE = 3,
+	RTE_DOZING = 5,
+	RTE_SLEEP = 6
+};
+
 /**
  * Keepalive failure callback.
  *
diff --git a/lib/librte_eal/common/rte_keepalive.c b/lib/librte_eal/common/rte_keepalive.c
index 23363ec..dd18e9f 100644
--- a/lib/librte_eal/common/rte_keepalive.c
+++ b/lib/librte_eal/common/rte_keepalive.c
@@ -42,12 +42,8 @@
 
 struct rte_keepalive {
 	/** Core Liveness. */
-	enum rte_keepalive_state {
-		ALIVE = 1,
-		MISSING = 0,
-		DEAD = 2,
-		GONE = 3
-	} __rte_cache_aligned state_flags[RTE_KEEPALIVE_MAXCORES];
+	enum rte_keepalive_state __rte_cache_aligned state_flags[
+		RTE_KEEPALIVE_MAXCORES];
 
 	/** Last-seen-alive timestamps */
 	uint64_t last_alive[RTE_KEEPALIVE_MAXCORES];
@@ -92,16 +88,18 @@ rte_keepalive_dispatch_pings(__rte_unused void *ptr_timer,
 			continue;
 
 		switch (keepcfg->state_flags[idx_core]) {
-		case ALIVE: /* Alive */
-			keepcfg->state_flags[idx_core] = MISSING;
+		case RTE_UNUSED:
+			break;
+		case RTE_ALIVE: /* Alive */
+			keepcfg->state_flags[idx_core] = RTE_MISSING;
 			keepcfg->last_alive[idx_core] = rte_rdtsc();
 			break;
-		case MISSING: /* MIA */
+		case RTE_MISSING: /* MIA */
 			print_trace("Core MIA. ", keepcfg, idx_core);
-			keepcfg->state_flags[idx_core] = DEAD;
+			keepcfg->state_flags[idx_core] = RTE_DEAD;
 			break;
-		case DEAD: /* Dead */
-			keepcfg->state_flags[idx_core] = GONE;
+		case RTE_DEAD: /* Dead */
+			keepcfg->state_flags[idx_core] = RTE_GONE;
 			print_trace("Core died. ", keepcfg, idx_core);
 			if (keepcfg->callback)
 				keepcfg->callback(
@@ -109,7 +107,13 @@ rte_keepalive_dispatch_pings(__rte_unused void *ptr_timer,
 					idx_core
 					);
 			break;
-		case GONE: /* Buried */
+		case RTE_GONE: /* Buried */
+			break;
+		case RTE_DOZING: /* Core going idle */
+			keepcfg->state_flags[idx_core] = RTE_SLEEP;
+			keepcfg->last_alive[idx_core] = rte_rdtsc();
+			break;
+		case RTE_SLEEP: /* Idled core */
 			break;
 		}
 	}
@@ -137,7 +141,7 @@ void
 rte_keepalive_register_core(struct rte_keepalive *keepcfg, const int id_core)
 {
 	if (id_core < RTE_KEEPALIVE_MAXCORES) {
-		keepcfg->active_cores[id_core] = 1;
+		keepcfg->active_cores[id_core] = RTE_ALIVE;
 		keepcfg->last_alive[id_core] = rte_rdtsc();
 	}
 }
@@ -145,5 +149,5 @@ rte_keepalive_register_core(struct rte_keepalive *keepcfg, const int id_core)
 void
 rte_keepalive_mark_alive(struct rte_keepalive *keepcfg)
 {
-	keepcfg->state_flags[rte_lcore_id()] = ALIVE;
+	keepcfg->state_flags[rte_lcore_id()] = RTE_ALIVE;
 }
-- 
2.5.5

[PATCH v3 2/3] eal: add additional keepalive callbacks

[Remy Horton]

Adds and documents new callbacks that allow transitions to core
states other than dead to be reported to applications.

Signed-off-by: Remy Horton <remy.horton@intel.com>
---
 doc/guides/rel_notes/release_16_07.rst          |  6 +++
 examples/Makefile                               |  2 +-
 lib/librte_eal/bsdapp/eal/rte_eal_version.map   |  1 +
 lib/librte_eal/common/include/rte_keepalive.h   | 51 +++++++++++++++++++++++++
 lib/librte_eal/common/rte_keepalive.c           | 30 +++++++++++++++
 lib/librte_eal/linuxapp/eal/rte_eal_version.map |  1 +
 6 files changed, 90 insertions(+), 1 deletion(-)

diff --git a/doc/guides/rel_notes/release_16_07.rst b/doc/guides/rel_notes/release_16_07.rst
index c0f6b02..df95321 100644
--- a/doc/guides/rel_notes/release_16_07.rst
+++ b/doc/guides/rel_notes/release_16_07.rst
@@ -66,6 +66,12 @@ New Features
   * Enable RSS per network interface through the configuration file.
   * Streamline the CLI code.
 
+* **Added keepalive enhancements.**
+
+  Adds support for reporting of core states other than dead to
+  monitoring applications, enabling the support of broader liveness
+  reporting to external processes.
+
 
 Resolved Issues
 ---------------
diff --git a/examples/Makefile b/examples/Makefile
index b28b30e..3bc635a 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,6 +1,6 @@
 #   BSD LICENSE
 #
-#   Copyright(c) 2014 6WIND S.A.
+#   Copyright(c) 2016 6WIND S.A.
 #
 #   Redistribution and use in source and binary forms, with or without
 #   modification, are permitted provided that the following conditions
diff --git a/lib/librte_eal/bsdapp/eal/rte_eal_version.map b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
index f8c3dea..0fb1cf2 100644
--- a/lib/librte_eal/bsdapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
@@ -156,5 +156,6 @@ DPDK_16.07 {
 	global:
 
 	pci_get_sysfs_path;
+	rte_keepalive_register_alive_callback;
 
 } DPDK_16.04;
diff --git a/lib/librte_eal/common/include/rte_keepalive.h b/lib/librte_eal/common/include/rte_keepalive.h
index 2f856c3..e04ea1c 100644
--- a/lib/librte_eal/common/include/rte_keepalive.h
+++ b/lib/librte_eal/common/include/rte_keepalive.h
@@ -63,12 +63,32 @@ enum rte_keepalive_state {
  *
  *  Receives a data pointer passed to rte_keepalive_create() and the id of the
  *  failed core.
+ *  @param data Data pointer passed to rte_keepalive_create()
+ *  @param id_core ID of the core that has failed
  */
 typedef void (*rte_keepalive_failure_callback_t)(
 	void *data,
 	const int id_core);
 
 /**
+ * Keepalive relay callback.
+ *
+ *  Receives a data pointer passed to rte_keepalive_register_relay_callback(),
+ *  the id of the core for which state is to be forwarded, and details of the
+ *  current core state.
+ *  @param data Data pointer passed to rte_keepalive_register_relay_callback()
+ *  @param id_core ID of the core for which state is being reported
+ *  @param core_state The current state of the core
+ *  @param Timestamp of when core was last seen alive
+ */
+typedef void (*rte_keepalive_relay_callback_t)(
+	void *data,
+	const int id_core,
+	enum rte_keepalive_state core_state,
+	uint64_t last_seen
+	);
+
+/**
  * Keepalive state structure.
  * @internal
  */
@@ -115,4 +135,35 @@ void rte_keepalive_register_core(struct rte_keepalive *keepcfg,
 void
 rte_keepalive_mark_alive(struct rte_keepalive *keepcfg);
 
+/**
+ * Per-core sleep-time indication.
+ * @param *keepcfg
+ *   Keepalive structure pointer
+ *
+ * If CPU idling is enabled, this function needs to be called from within
+ * the main process loop of the LCore going to sleep, in order to avoid
+ * the LCore being mis-detected as dead.
+ */
+void
+rte_keepalive_mark_sleep(struct rte_keepalive *keepcfg);
+
+/**
+ * Registers a 'live core' callback.
+ *
+ * The complement of the 'dead core' callback. This is called when a
+ * core is known to be alive, and is intended for cases when an app
+ * needs to know 'liveness' beyond just knowing when a core has died.
+ *
+ * @param *keepcfg
+ *   Keepalive structure pointer
+ * @param callback
+ *   Function called upon detection of a dead core.
+ * @param data
+ *   Data pointer to be passed to function callback.
+ */
+void
+rte_keepalive_register_relay_callback(struct rte_keepalive *keepcfg,
+	rte_keepalive_relay_callback_t callback,
+	void *data);
+
 #endif /* _KEEPALIVE_H_ */
diff --git a/lib/librte_eal/common/rte_keepalive.c b/lib/librte_eal/common/rte_keepalive.c
index dd18e9f..0aeee46 100644
--- a/lib/librte_eal/common/rte_keepalive.c
+++ b/lib/librte_eal/common/rte_keepalive.c
@@ -64,6 +64,15 @@ struct rte_keepalive {
 	void *callback_data;
 	uint64_t tsc_initial;
 	uint64_t tsc_mhz;
+
+	/** Core state relay handler. */
+	rte_keepalive_relay_callback_t relay_callback;
+
+	/**
+	 * Core state relay handler app data.
+	 * Pointer is passed to live core handler.
+	 */
+	void *relay_callback_data;
 };
 
 static void
@@ -116,6 +125,13 @@ rte_keepalive_dispatch_pings(__rte_unused void *ptr_timer,
 		case RTE_SLEEP: /* Idled core */
 			break;
 		}
+		if (keepcfg->relay_callback)
+			keepcfg->relay_callback(
+				keepcfg->relay_callback_data,
+				idx_core,
+				keepcfg->state_flags[idx_core],
+				keepcfg->last_alive[idx_core]
+				);
 	}
 }
 
@@ -137,6 +153,14 @@ rte_keepalive_create(rte_keepalive_failure_callback_t callback,
 	return keepcfg;
 }
 
+void rte_keepalive_register_relay_callback(struct rte_keepalive *keepcfg,
+	rte_keepalive_relay_callback_t callback,
+	void *data)
+{
+	keepcfg->relay_callback = callback;
+	keepcfg->relay_callback_data = data;
+}
+
 void
 rte_keepalive_register_core(struct rte_keepalive *keepcfg, const int id_core)
 {
@@ -151,3 +175,9 @@ rte_keepalive_mark_alive(struct rte_keepalive *keepcfg)
 {
 	keepcfg->state_flags[rte_lcore_id()] = RTE_ALIVE;
 }
+
+void
+rte_keepalive_mark_sleep(struct rte_keepalive *keepcfg)
+{
+	keepcfg->state_flags[rte_lcore_id()] = RTE_DOZING;
+}
diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
index 3d0ff93..34f44a7 100644
--- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
@@ -159,5 +159,6 @@ DPDK_16.07 {
 	global:
 
 	pci_get_sysfs_path;
+	rte_keepalive_register_alive_callback;
 
 } DPDK_16.04;
-- 
2.5.5

[PATCH v3 3/3] examples/l2fwd-keepalive: add IPC liveness reporting

[Remy Horton]

Changes the l2fwd keepalive example to show how the new keepalive
enhancements can be used to relay core state to an external process.

Signed-off-by: Remy Horton <remy.horton@intel.com>
---
 examples/Makefile                          |   1 +
 examples/l2fwd-keepalive/Makefile          |   4 +-
 examples/l2fwd-keepalive/ka-agent/Makefile |  48 +++++++++
 examples/l2fwd-keepalive/ka-agent/main.c   | 150 +++++++++++++++++++++++++++++
 examples/l2fwd-keepalive/main.c            |  22 ++++-
 examples/l2fwd-keepalive/shm.c             | 128 ++++++++++++++++++++++++
 examples/l2fwd-keepalive/shm.h             |  89 +++++++++++++++++
 7 files changed, 438 insertions(+), 4 deletions(-)
 create mode 100644 examples/l2fwd-keepalive/ka-agent/Makefile
 create mode 100644 examples/l2fwd-keepalive/ka-agent/main.c
 create mode 100644 examples/l2fwd-keepalive/shm.c
 create mode 100644 examples/l2fwd-keepalive/shm.h

diff --git a/examples/Makefile b/examples/Makefile
index 3bc635a..f650d3e 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -64,6 +64,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += l2fwd-crypto
 DIRS-$(CONFIG_RTE_LIBRTE_IVSHMEM) += l2fwd-ivshmem
 DIRS-$(CONFIG_RTE_LIBRTE_JOBSTATS) += l2fwd-jobstats
 DIRS-y += l2fwd-keepalive
+DIRS-y += l2fwd-keepalive/ka-agent
 DIRS-$(CONFIG_RTE_LIBRTE_LPM) += l3fwd
 DIRS-$(CONFIG_RTE_LIBRTE_ACL) += l3fwd-acl
 ifeq ($(CONFIG_RTE_LIBRTE_LPM),y)
diff --git a/examples/l2fwd-keepalive/Makefile b/examples/l2fwd-keepalive/Makefile
index 568edcb..3fcf513 100644
--- a/examples/l2fwd-keepalive/Makefile
+++ b/examples/l2fwd-keepalive/Makefile
@@ -1,6 +1,6 @@
 #   BSD LICENSE
 #
-#   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+#   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
 #   All rights reserved.
 #
 #   Redistribution and use in source and binary forms, with or without
@@ -42,7 +42,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
 APP = l2fwd-keepalive
 
 # all source are stored in SRCS-y
-SRCS-y := main.c
+SRCS-y := main.c shm.c
 
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
diff --git a/examples/l2fwd-keepalive/ka-agent/Makefile b/examples/l2fwd-keepalive/ka-agent/Makefile
new file mode 100644
index 0000000..f008428
--- /dev/null
+++ b/examples/l2fwd-keepalive/ka-agent/Makefile
@@ -0,0 +1,48 @@
+#   BSD LICENSE
+#
+#   Copyright(c) 2016 Intel Corporation. All rights reserved.
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Intel Corporation nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overridden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# binary name
+APP = ka-agent
+
+# all source are stored in SRCS-y
+SRCS-y := main.c
+
+CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR)/../
+include $(RTE_SDK)/mk/rte.extapp.mk
diff --git a/examples/l2fwd-keepalive/ka-agent/main.c b/examples/l2fwd-keepalive/ka-agent/main.c
new file mode 100644
index 0000000..d1f5e6d
--- /dev/null
+++ b/examples/l2fwd-keepalive/ka-agent/main.c
@@ -0,0 +1,150 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/wait.h>
+#include <sys/queue.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <time.h>
+
+#include <rte_keepalive.h>
+
+#include <shm.h>
+
+#define MAX_TIMEOUTS 4
+#define SEM_TIMEOUT_SECS 2
+
+static struct rte_keepalive_shm *ka_shm_create(void)
+{
+	int fd = shm_open(RTE_KEEPALIVE_SHM_NAME, O_RDWR, 0666);
+	size_t size = sizeof(struct rte_keepalive_shm);
+	struct rte_keepalive_shm *shm;
+
+	if (fd < 0)
+		printf("Failed to open %s as SHM:%s\n",
+			RTE_KEEPALIVE_SHM_NAME,
+		strerror(errno));
+	else {
+		shm = (struct rte_keepalive_shm *) mmap(
+			0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+		close(fd);
+		if (shm == MAP_FAILED)
+			printf("Failed to mmap SHM:%s\n", strerror(errno));
+		else
+			return shm;
+	}
+
+	/* Reset to zero, as it was set to MAP_FAILED aka: (void *)-1 */
+	shm = 0;
+	return NULL;
+}
+
+int main(void)
+{
+	struct rte_keepalive_shm *shm = ka_shm_create();
+	struct timespec timeout = { .tv_nsec = 0 };
+	int idx_core;
+	int cnt_cores;
+	uint64_t last_seen_alive_time = 0;
+	uint64_t most_recent_alive_time;
+	int cnt_timeouts = 0;
+	int sem_errno;
+
+	if (shm == NULL) {
+		printf("Unable to access shared core state\n");
+		return 1;
+	}
+	while (1) {
+		most_recent_alive_time = 0;
+		for (idx_core = 0; idx_core < RTE_KEEPALIVE_MAXCORES;
+				idx_core++)
+			if (shm->core_last_seen_times[idx_core] >
+					most_recent_alive_time)
+				most_recent_alive_time =
+					shm->core_last_seen_times[idx_core];
+
+		timeout.tv_sec = time(NULL) + SEM_TIMEOUT_SECS;
+		if (sem_timedwait(&shm->core_died, &timeout) == -1) {
+			/* Assume no core death signals and no change in any
+			 * last-seen times is the keepalive monitor itself
+			 * failing.
+			 */
+			sem_errno = errno;
+			last_seen_alive_time = most_recent_alive_time;
+			if (sem_errno == ETIMEDOUT) {
+				if (last_seen_alive_time ==
+						most_recent_alive_time &&
+						cnt_timeouts++ >
+						MAX_TIMEOUTS) {
+					printf("No updates. Exiting..\n");
+					break;
+					}
+			} else
+				printf("sem_timedwait() error (%s)\n",
+					strerror(sem_errno));
+			continue;
+		}
+		cnt_timeouts = 0;
+
+		cnt_cores = 0;
+		for (idx_core = 0; idx_core < RTE_KEEPALIVE_MAXCORES;
+				idx_core++)
+			if (shm->core_state[idx_core] == RTE_DEAD)
+				cnt_cores++;
+		if (cnt_cores == 0) {
+			/* Can happen if core was restarted since Semaphore
+			 * was sent, due to agent being offline.
+			 */
+			printf("Warning: Empty dead core report\n");
+			continue;
+		}
+
+		printf("%i dead cores: ", cnt_cores);
+		for (idx_core = 0;
+				idx_core < RTE_KEEPALIVE_MAXCORES;
+				idx_core++)
+			if (shm->core_state[idx_core] == RTE_DEAD)
+				printf("%d, ", idx_core);
+		printf("\b\b\n");
+	}
+	if (munmap(shm, sizeof(struct rte_keepalive_shm)) != 0)
+		printf("Warning: munmap() failed\n");
+	return 0;
+}
diff --git a/examples/l2fwd-keepalive/main.c b/examples/l2fwd-keepalive/main.c
index 4a0d9b6..84a59eb 100644
--- a/examples/l2fwd-keepalive/main.c
+++ b/examples/l2fwd-keepalive/main.c
@@ -72,6 +72,8 @@
 #include <rte_timer.h>
 #include <rte_keepalive.h>
 
+#include "shm.h"
+
 #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1
 
 #define NB_MBUF   8192
@@ -523,7 +525,7 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask)
 }
 
 static void
-dead_core(__attribute__((unused)) void *ptr_data, const int id_core)
+dead_core(__rte_unused void *ptr_data, const int id_core)
 {
 	printf("Dead core %i - restarting..\n", id_core);
 	if (rte_eal_get_lcore_state(id_core) == FINISHED) {
@@ -534,6 +536,14 @@ dead_core(__attribute__((unused)) void *ptr_data, const int id_core)
 	}
 }
 
+static void
+relay_core_state(void *ptr_data, const int id_core,
+	const enum rte_keepalive_state core_state, uint64_t last_alive)
+{
+	rte_keepalive_relayed_state((struct rte_keepalive_shm *)ptr_data,
+		id_core, core_state, last_alive);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -722,10 +732,18 @@ main(int argc, char **argv)
 	rte_timer_init(&stats_timer);
 
 	if (check_period > 0) {
+		struct rte_keepalive_shm *ka_shm;
+
+		ka_shm = rte_keepalive_shm_create();
+		if (ka_shm == NULL)
+			rte_exit(EXIT_FAILURE,
+				"rte_keepalive_shm_create() failed");
 		rte_global_keepalive_info =
-			rte_keepalive_create(&dead_core, NULL);
+			rte_keepalive_create(&dead_core, ka_shm);
 		if (rte_global_keepalive_info == NULL)
 			rte_exit(EXIT_FAILURE, "init_keep_alive() failed");
+		rte_keepalive_register_relay_callback(rte_global_keepalive_info,
+			relay_core_state, ka_shm);
 		rte_timer_init(&hb_timer);
 		if (rte_timer_reset(&hb_timer,
 				(check_period * rte_get_timer_hz()) / 1000,
diff --git a/examples/l2fwd-keepalive/shm.c b/examples/l2fwd-keepalive/shm.c
new file mode 100644
index 0000000..1a988c6
--- /dev/null
+++ b/examples/l2fwd-keepalive/shm.c
@@ -0,0 +1,128 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <time.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_keepalive.h>
+
+#include "shm.h"
+
+struct rte_keepalive_shm *rte_keepalive_shm_create(void)
+{
+	int fd;
+	int idx_core;
+	struct rte_keepalive_shm *ka_shm;
+
+	/* If any existing object is not unlinked, it makes it all too easy
+	 * for clients to end up with stale shared memory blocks when
+	 * restarted. Unlinking makes sure subsequent shm_open by clients
+	 * will get the new block mapped below.
+	 */
+	if (shm_unlink(RTE_KEEPALIVE_SHM_NAME) == -1 && errno != ENOENT)
+		printf("Warning: Error unlinking stale %s (%s)\n",
+			RTE_KEEPALIVE_SHM_NAME, strerror(errno));
+
+	fd = shm_open(RTE_KEEPALIVE_SHM_NAME,
+		O_CREAT | O_TRUNC | O_RDWR, 0666);
+	if (fd < 0)
+		RTE_LOG(INFO, EAL,
+			"Failed to open %s as SHM (%s)\n",
+			RTE_KEEPALIVE_SHM_NAME,
+			strerror(errno));
+	else if (ftruncate(fd, sizeof(struct rte_keepalive_shm)) != 0)
+		RTE_LOG(INFO, EAL,
+			"Failed to resize SHM (%s)\n", strerror(errno));
+	else {
+		ka_shm = (struct rte_keepalive_shm *) mmap(
+			0, sizeof(struct rte_keepalive_shm),
+			PROT_READ | PROT_WRITE,	MAP_SHARED, fd, 0);
+		close(fd);
+		if (ka_shm == MAP_FAILED)
+			RTE_LOG(INFO, EAL,
+				"Failed to mmap SHM (%s)\n", strerror(errno));
+		else {
+			memset(ka_shm, 0, sizeof(struct rte_keepalive_shm));
+
+			/* Initialize the semaphores for IPC/SHM use */
+			if (sem_init(&ka_shm->core_died, 1, 0) != 0) {
+				RTE_LOG(INFO, EAL,
+					"Failed to setup SHM semaphore (%s)\n",
+					strerror(errno));
+				return NULL;
+			}
+
+			/* Set all cores to 'not present' */
+			for (idx_core = 0;
+					idx_core < RTE_KEEPALIVE_MAXCORES;
+					idx_core++) {
+				ka_shm->core_state[idx_core] = RTE_UNUSED;
+				ka_shm->core_last_seen_times[idx_core] = 0;
+			}
+
+			return ka_shm;
+		}
+	}
+return NULL;
+}
+
+void rte_keepalive_relayed_state(struct rte_keepalive_shm *shm,
+	const int id_core, const enum rte_keepalive_state core_state,
+	__rte_unused uint64_t last_alive)
+{
+	int count;
+
+	shm->core_state[id_core] = core_state;
+	shm->core_last_seen_times[id_core] = last_alive;
+
+	if (core_state == RTE_KEEPALIVE_SHM_DEAD) {
+		/* Since core has died, also signal ka_agent.
+		 *
+		 * Limit number of times semaphore can be incremented, in case
+		 * ka_agent is not active.
+		 */
+		if (sem_getvalue(&shm->core_died, &count) == -1) {
+			RTE_LOG(INFO, EAL, "Semaphore check failed(%s)\n",
+				strerror(errno));
+			return;
+		}
+		if (count > 1)
+			return;
+
+		if (sem_post(&shm->core_died) != 0)
+			RTE_LOG(INFO, EAL,
+				"Failed to increment semaphore (%s)\n",
+				strerror(errno));
+	}
+}
diff --git a/examples/l2fwd-keepalive/shm.h b/examples/l2fwd-keepalive/shm.h
new file mode 100644
index 0000000..25e1b61
--- /dev/null
+++ b/examples/l2fwd-keepalive/shm.h
@@ -0,0 +1,89 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define RTE_KEEPALIVE_SHM_NAME "/dpdk_keepalive_shm_name"
+
+#define RTE_KEEPALIVE_SHM_ALIVE 1
+#define RTE_KEEPALIVE_SHM_DEAD 2
+
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <semaphore.h>
+#include <rte_keepalive.h>
+
+/**
+ * Keepalive SHM structure.
+ *
+ * The shared memory allocated by the primary is this size, and contains the
+ * information as contained within this struct. A secondary may open the SHM,
+ * and read the contents.
+ */
+struct rte_keepalive_shm {
+	/** IPC semaphore. Posted when a core dies */
+	sem_t core_died;
+
+	/**
+	 * Relayed status of each core.
+	 */
+	enum rte_keepalive_state core_state[RTE_KEEPALIVE_MAXCORES];
+
+	/**
+	 * Last-seen-alive timestamps for the cores
+	 */
+	uint64_t core_last_seen_times[RTE_KEEPALIVE_MAXCORES];
+};
+
+/**
+ * Create shared host memory keepalive object.
+ * @return
+ *  Pointer to SHM keepalive structure, or NULL on failure.
+ */
+struct rte_keepalive_shm *rte_keepalive_shm_create(void);
+
+/**
+ * Relays state for given core
+ * @param *shm
+ *  Pointer to SHM keepalive structure.
+ * @param id_core
+ *  Id of core
+ * @param core_state
+ *  State of core
+ * @param last_alive
+ *  Last seen timestamp for core
+ */
+void rte_keepalive_relayed_state(struct rte_keepalive_shm *shm,
+	const int id_core, const enum rte_keepalive_state core_state,
+	uint64_t last_alive);
-- 
2.5.5

Re: [PATCH v3 1/3] eal: export keepalive state enumerations

[Thomas Monjalon]

2016-06-15 10:17, Remy Horton:
> +enum rte_keepalive_state {
> +	RTE_UNUSED = 0,
> +	RTE_ALIVE = 1,
> +	RTE_MISSING = 4,
> +	RTE_DEAD = 2,
> +	RTE_GONE = 3,
> +	RTE_DOZING = 5,
> +	RTE_SLEEP = 6
> +};

I'm concerned about the namespace here.
RTE_UNUSED and others have a chance to not be unique enough.
Is it possible to have a longer prefix like RTE_KA_ or RTE_STATE_
or RTE_KA_STATE_?

Re: [PATCH v3 1/3] eal: export keepalive state enumerations

[Remy Horton]

On 15/06/2016 10:27, Thomas Monjalon wrote:
[..]
> I'm concerned about the namespace here.
> RTE_UNUSED and others have a chance to not be unique enough.
> Is it possible to have a longer prefix like RTE_KA_ or RTE_STATE_
> or RTE_KA_STATE_?
>

Good point - I'll go with RTE_KA_STATE_ as it is most descriptive.

..Remy

[PATCH v4 0/3] Keep-alive enhancements

[Remy Horton]

This patchset adds enhancements to the keepalive core monitoring and
reporting sub-system. The first is support for idled (sleeping and
frequency-stepped) CPU cores, and the second is support for applications
to be notified of active as well as faulted cores. The latter is to allow
core state to be relayed to external (secondary) processes, which is
demonstrated by changes to the l2fwd-keepalive example.

--

v4 changes
* Use RTE_KA_STATE_ prefix instead of RTE_ (namespace issues)

v3 changes
* Rebased to master
* Exposed keepalive state enumerations
* Changed keepalive state enumerations to use RTE_ prefix
* Added missing parameter documentation
* Doc changes squashed

v2 changes:
* Some date & typos fixups
* State enum made public and extended with new states
* Generalised 'alive' callback to all states
* Last-alive shows gone-to-sleep time for idle cores
* Removed some redundant sanity checks
* Last-alive time exposed to application
* #define'd semaphore timeout
* Agent checks for dead keepalive

Remy Horton (3):
  eal: export keepalive state enumerations
  eal: add additional keepalive callbacks
  examples/l2fwd-keepalive: add IPC liveness reporting

 doc/guides/rel_notes/release_16_07.rst          |   6 +
 examples/Makefile                               |   3 +-
 examples/l2fwd-keepalive/Makefile               |   5 +-
 examples/l2fwd-keepalive/ka-agent/Makefile      |  49 ++++++++
 examples/l2fwd-keepalive/ka-agent/main.c        | 150 ++++++++++++++++++++++++
 examples/l2fwd-keepalive/main.c                 |  22 +++-
 examples/l2fwd-keepalive/shm.c                  | 129 ++++++++++++++++++++
 examples/l2fwd-keepalive/shm.h                  |  89 ++++++++++++++
 lib/librte_eal/bsdapp/eal/rte_eal_version.map   |   1 +
 lib/librte_eal/common/include/rte_keepalive.h   |  63 +++++++++-
 lib/librte_eal/common/rte_keepalive.c           |  64 +++++++---
 lib/librte_eal/linuxapp/eal/rte_eal_version.map |   2 +
 12 files changed, 562 insertions(+), 21 deletions(-)
 create mode 100644 examples/l2fwd-keepalive/ka-agent/Makefile
 create mode 100644 examples/l2fwd-keepalive/ka-agent/main.c
 create mode 100644 examples/l2fwd-keepalive/shm.c
 create mode 100644 examples/l2fwd-keepalive/shm.h

-- 
2.5.5

[PATCH v4 1/3] eal: export keepalive state enumerations

[Remy Horton]

Changes the keepalive state from an anonymous enum to a declared one
which is externally visible, so that keepalive enum values can be
used by applications.

Signed-off-by: Remy Horton <remy.horton@intel.com>
---
 lib/librte_eal/common/include/rte_keepalive.h | 12 +++++++++-
 lib/librte_eal/common/rte_keepalive.c         | 34 +++++++++++++++------------
 2 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/lib/librte_eal/common/include/rte_keepalive.h b/lib/librte_eal/common/include/rte_keepalive.h
index 10dac2e..d01a654 100644
--- a/lib/librte_eal/common/include/rte_keepalive.h
+++ b/lib/librte_eal/common/include/rte_keepalive.h
@@ -1,7 +1,7 @@
 /*-
  *   BSD LICENSE
  *
- *   Copyright 2015 Intel Shannon Ltd. All rights reserved.
+ *   Copyright 2015-2016 Intel Shannon Ltd. All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
  *   modification, are permitted provided that the following conditions
@@ -48,6 +48,16 @@
 #define RTE_KEEPALIVE_MAXCORES RTE_MAX_LCORE
 #endif
 
+enum rte_keepalive_state {
+	RTE_KA_STATE_UNUSED = 0,
+	RTE_KA_STATE_ALIVE = 1,
+	RTE_KA_STATE_MISSING = 4,
+	RTE_KA_STATE_DEAD = 2,
+	RTE_KA_STATE_GONE = 3,
+	RTE_KA_STATE_DOZING = 5,
+	RTE_KA_STATE_SLEEP = 6
+};
+
 /**
  * Keepalive failure callback.
  *
diff --git a/lib/librte_eal/common/rte_keepalive.c b/lib/librte_eal/common/rte_keepalive.c
index 23363ec..8b14370 100644
--- a/lib/librte_eal/common/rte_keepalive.c
+++ b/lib/librte_eal/common/rte_keepalive.c
@@ -42,12 +42,8 @@
 
 struct rte_keepalive {
 	/** Core Liveness. */
-	enum rte_keepalive_state {
-		ALIVE = 1,
-		MISSING = 0,
-		DEAD = 2,
-		GONE = 3
-	} __rte_cache_aligned state_flags[RTE_KEEPALIVE_MAXCORES];
+	enum rte_keepalive_state __rte_cache_aligned state_flags[
+		RTE_KEEPALIVE_MAXCORES];
 
 	/** Last-seen-alive timestamps */
 	uint64_t last_alive[RTE_KEEPALIVE_MAXCORES];
@@ -92,16 +88,18 @@ rte_keepalive_dispatch_pings(__rte_unused void *ptr_timer,
 			continue;
 
 		switch (keepcfg->state_flags[idx_core]) {
-		case ALIVE: /* Alive */
-			keepcfg->state_flags[idx_core] = MISSING;
+		case RTE_KA_STATE_UNUSED:
+			break;
+		case RTE_KA_STATE_ALIVE: /* Alive */
+			keepcfg->state_flags[idx_core] = RTE_KA_STATE_MISSING;
 			keepcfg->last_alive[idx_core] = rte_rdtsc();
 			break;
-		case MISSING: /* MIA */
+		case RTE_KA_STATE_MISSING: /* MIA */
 			print_trace("Core MIA. ", keepcfg, idx_core);
-			keepcfg->state_flags[idx_core] = DEAD;
+			keepcfg->state_flags[idx_core] = RTE_KA_STATE_DEAD;
 			break;
-		case DEAD: /* Dead */
-			keepcfg->state_flags[idx_core] = GONE;
+		case RTE_KA_STATE_DEAD: /* Dead */
+			keepcfg->state_flags[idx_core] = RTE_KA_STATE_GONE;
 			print_trace("Core died. ", keepcfg, idx_core);
 			if (keepcfg->callback)
 				keepcfg->callback(
@@ -109,7 +107,13 @@ rte_keepalive_dispatch_pings(__rte_unused void *ptr_timer,
 					idx_core
 					);
 			break;
-		case GONE: /* Buried */
+		case RTE_KA_STATE_GONE: /* Buried */
+			break;
+		case RTE_KA_STATE_DOZING: /* Core going idle */
+			keepcfg->state_flags[idx_core] = RTE_KA_STATE_SLEEP;
+			keepcfg->last_alive[idx_core] = rte_rdtsc();
+			break;
+		case RTE_KA_STATE_SLEEP: /* Idled core */
 			break;
 		}
 	}
@@ -137,7 +141,7 @@ void
 rte_keepalive_register_core(struct rte_keepalive *keepcfg, const int id_core)
 {
 	if (id_core < RTE_KEEPALIVE_MAXCORES) {
-		keepcfg->active_cores[id_core] = 1;
+		keepcfg->active_cores[id_core] = RTE_KA_STATE_ALIVE;
 		keepcfg->last_alive[id_core] = rte_rdtsc();
 	}
 }
@@ -145,5 +149,5 @@ rte_keepalive_register_core(struct rte_keepalive *keepcfg, const int id_core)
 void
 rte_keepalive_mark_alive(struct rte_keepalive *keepcfg)
 {
-	keepcfg->state_flags[rte_lcore_id()] = ALIVE;
+	keepcfg->state_flags[rte_lcore_id()] = RTE_KA_STATE_ALIVE;
 }
-- 
2.5.5

[PATCH v4 2/3] eal: add additional keepalive callbacks

[Remy Horton]

Adds and documents new callbacks that allow transitions to core
states other than dead to be reported to applications.

Signed-off-by: Remy Horton <remy.horton@intel.com>
---
 doc/guides/rel_notes/release_16_07.rst          |  6 +++
 examples/Makefile                               |  2 +-
 lib/librte_eal/bsdapp/eal/rte_eal_version.map   |  1 +
 lib/librte_eal/common/include/rte_keepalive.h   | 51 +++++++++++++++++++++++++
 lib/librte_eal/common/rte_keepalive.c           | 30 +++++++++++++++
 lib/librte_eal/linuxapp/eal/rte_eal_version.map |  2 +
 6 files changed, 91 insertions(+), 1 deletion(-)

diff --git a/doc/guides/rel_notes/release_16_07.rst b/doc/guides/rel_notes/release_16_07.rst
index c0f6b02..df95321 100644
--- a/doc/guides/rel_notes/release_16_07.rst
+++ b/doc/guides/rel_notes/release_16_07.rst
@@ -66,6 +66,12 @@ New Features
   * Enable RSS per network interface through the configuration file.
   * Streamline the CLI code.
 
+* **Added keepalive enhancements.**
+
+  Adds support for reporting of core states other than dead to
+  monitoring applications, enabling the support of broader liveness
+  reporting to external processes.
+
 
 Resolved Issues
 ---------------
diff --git a/examples/Makefile b/examples/Makefile
index b28b30e..3bc635a 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -1,6 +1,6 @@
 #   BSD LICENSE
 #
-#   Copyright(c) 2014 6WIND S.A.
+#   Copyright(c) 2016 6WIND S.A.
 #
 #   Redistribution and use in source and binary forms, with or without
 #   modification, are permitted provided that the following conditions
diff --git a/lib/librte_eal/bsdapp/eal/rte_eal_version.map b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
index f8c3dea..0fb1cf2 100644
--- a/lib/librte_eal/bsdapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
@@ -156,5 +156,6 @@ DPDK_16.07 {
 	global:
 
 	pci_get_sysfs_path;
+	rte_keepalive_register_alive_callback;
 
 } DPDK_16.04;
diff --git a/lib/librte_eal/common/include/rte_keepalive.h b/lib/librte_eal/common/include/rte_keepalive.h
index d01a654..88ad8e4 100644
--- a/lib/librte_eal/common/include/rte_keepalive.h
+++ b/lib/librte_eal/common/include/rte_keepalive.h
@@ -63,12 +63,32 @@ enum rte_keepalive_state {
  *
  *  Receives a data pointer passed to rte_keepalive_create() and the id of the
  *  failed core.
+ *  @param data Data pointer passed to rte_keepalive_create()
+ *  @param id_core ID of the core that has failed
  */
 typedef void (*rte_keepalive_failure_callback_t)(
 	void *data,
 	const int id_core);
 
 /**
+ * Keepalive relay callback.
+ *
+ *  Receives a data pointer passed to rte_keepalive_register_relay_callback(),
+ *  the id of the core for which state is to be forwarded, and details of the
+ *  current core state.
+ *  @param data Data pointer passed to rte_keepalive_register_relay_callback()
+ *  @param id_core ID of the core for which state is being reported
+ *  @param core_state The current state of the core
+ *  @param Timestamp of when core was last seen alive
+ */
+typedef void (*rte_keepalive_relay_callback_t)(
+	void *data,
+	const int id_core,
+	enum rte_keepalive_state core_state,
+	uint64_t last_seen
+	);
+
+/**
  * Keepalive state structure.
  * @internal
  */
@@ -115,4 +135,35 @@ void rte_keepalive_register_core(struct rte_keepalive *keepcfg,
 void
 rte_keepalive_mark_alive(struct rte_keepalive *keepcfg);
 
+/**
+ * Per-core sleep-time indication.
+ * @param *keepcfg
+ *   Keepalive structure pointer
+ *
+ * If CPU idling is enabled, this function needs to be called from within
+ * the main process loop of the LCore going to sleep, in order to avoid
+ * the LCore being mis-detected as dead.
+ */
+void
+rte_keepalive_mark_sleep(struct rte_keepalive *keepcfg);
+
+/**
+ * Registers a 'live core' callback.
+ *
+ * The complement of the 'dead core' callback. This is called when a
+ * core is known to be alive, and is intended for cases when an app
+ * needs to know 'liveness' beyond just knowing when a core has died.
+ *
+ * @param *keepcfg
+ *   Keepalive structure pointer
+ * @param callback
+ *   Function called upon detection of a dead core.
+ * @param data
+ *   Data pointer to be passed to function callback.
+ */
+void
+rte_keepalive_register_relay_callback(struct rte_keepalive *keepcfg,
+	rte_keepalive_relay_callback_t callback,
+	void *data);
+
 #endif /* _KEEPALIVE_H_ */
diff --git a/lib/librte_eal/common/rte_keepalive.c b/lib/librte_eal/common/rte_keepalive.c
index 8b14370..9765d1b 100644
--- a/lib/librte_eal/common/rte_keepalive.c
+++ b/lib/librte_eal/common/rte_keepalive.c
@@ -64,6 +64,15 @@ struct rte_keepalive {
 	void *callback_data;
 	uint64_t tsc_initial;
 	uint64_t tsc_mhz;
+
+	/** Core state relay handler. */
+	rte_keepalive_relay_callback_t relay_callback;
+
+	/**
+	 * Core state relay handler app data.
+	 * Pointer is passed to live core handler.
+	 */
+	void *relay_callback_data;
 };
 
 static void
@@ -116,6 +125,13 @@ rte_keepalive_dispatch_pings(__rte_unused void *ptr_timer,
 		case RTE_KA_STATE_SLEEP: /* Idled core */
 			break;
 		}
+		if (keepcfg->relay_callback)
+			keepcfg->relay_callback(
+				keepcfg->relay_callback_data,
+				idx_core,
+				keepcfg->state_flags[idx_core],
+				keepcfg->last_alive[idx_core]
+				);
 	}
 }
 
@@ -137,6 +153,14 @@ rte_keepalive_create(rte_keepalive_failure_callback_t callback,
 	return keepcfg;
 }
 
+void rte_keepalive_register_relay_callback(struct rte_keepalive *keepcfg,
+	rte_keepalive_relay_callback_t callback,
+	void *data)
+{
+	keepcfg->relay_callback = callback;
+	keepcfg->relay_callback_data = data;
+}
+
 void
 rte_keepalive_register_core(struct rte_keepalive *keepcfg, const int id_core)
 {
@@ -151,3 +175,9 @@ rte_keepalive_mark_alive(struct rte_keepalive *keepcfg)
 {
 	keepcfg->state_flags[rte_lcore_id()] = RTE_KA_STATE_ALIVE;
 }
+
+void
+rte_keepalive_mark_sleep(struct rte_keepalive *keepcfg)
+{
+	keepcfg->state_flags[rte_lcore_id()] = RTE_KA_STATE_DOZING;
+}
diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
index 3d0ff93..cacf9e1 100644
--- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
@@ -159,5 +159,7 @@ DPDK_16.07 {
 	global:
 
 	pci_get_sysfs_path;
+	rte_keepalive_register_alive_callback;
+	rte_keepalive_register_relay_callback;
 
 } DPDK_16.04;
-- 
2.5.5

[PATCH v4 3/3] examples/l2fwd-keepalive: add IPC liveness reporting

[Remy Horton]

Changes the l2fwd keepalive example to show how the new keepalive
enhancements can be used to relay core state to an external process.

Signed-off-by: Remy Horton <remy.horton@intel.com>
---
 examples/Makefile                          |   1 +
 examples/l2fwd-keepalive/Makefile          |   5 +-
 examples/l2fwd-keepalive/ka-agent/Makefile |  49 ++++++++++
 examples/l2fwd-keepalive/ka-agent/main.c   | 150 +++++++++++++++++++++++++++++
 examples/l2fwd-keepalive/main.c            |  22 ++++-
 examples/l2fwd-keepalive/shm.c             | 129 +++++++++++++++++++++++++
 examples/l2fwd-keepalive/shm.h             |  89 +++++++++++++++++
 7 files changed, 441 insertions(+), 4 deletions(-)
 create mode 100644 examples/l2fwd-keepalive/ka-agent/Makefile
 create mode 100644 examples/l2fwd-keepalive/ka-agent/main.c
 create mode 100644 examples/l2fwd-keepalive/shm.c
 create mode 100644 examples/l2fwd-keepalive/shm.h

diff --git a/examples/Makefile b/examples/Makefile
index 3bc635a..f650d3e 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -64,6 +64,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += l2fwd-crypto
 DIRS-$(CONFIG_RTE_LIBRTE_IVSHMEM) += l2fwd-ivshmem
 DIRS-$(CONFIG_RTE_LIBRTE_JOBSTATS) += l2fwd-jobstats
 DIRS-y += l2fwd-keepalive
+DIRS-y += l2fwd-keepalive/ka-agent
 DIRS-$(CONFIG_RTE_LIBRTE_LPM) += l3fwd
 DIRS-$(CONFIG_RTE_LIBRTE_ACL) += l3fwd-acl
 ifeq ($(CONFIG_RTE_LIBRTE_LPM),y)
diff --git a/examples/l2fwd-keepalive/Makefile b/examples/l2fwd-keepalive/Makefile
index 568edcb..ca45a79 100644
--- a/examples/l2fwd-keepalive/Makefile
+++ b/examples/l2fwd-keepalive/Makefile
@@ -1,6 +1,6 @@
 #   BSD LICENSE
 #
-#   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+#   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
 #   All rights reserved.
 #
 #   Redistribution and use in source and binary forms, with or without
@@ -42,9 +42,10 @@ include $(RTE_SDK)/mk/rte.vars.mk
 APP = l2fwd-keepalive
 
 # all source are stored in SRCS-y
-SRCS-y := main.c
+SRCS-y := main.c shm.c
 
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
+LDFLAGS += -lrt
 
 include $(RTE_SDK)/mk/rte.extapp.mk
diff --git a/examples/l2fwd-keepalive/ka-agent/Makefile b/examples/l2fwd-keepalive/ka-agent/Makefile
new file mode 100644
index 0000000..fd0c38b
--- /dev/null
+++ b/examples/l2fwd-keepalive/ka-agent/Makefile
@@ -0,0 +1,49 @@
+#   BSD LICENSE
+#
+#   Copyright(c) 2016 Intel Corporation. All rights reserved.
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Intel Corporation nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overridden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# binary name
+APP = ka-agent
+
+# all source are stored in SRCS-y
+SRCS-y := main.c
+
+CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR)/../
+LDFLAGS += -lrt
+include $(RTE_SDK)/mk/rte.extapp.mk
diff --git a/examples/l2fwd-keepalive/ka-agent/main.c b/examples/l2fwd-keepalive/ka-agent/main.c
new file mode 100644
index 0000000..be1c7f4
--- /dev/null
+++ b/examples/l2fwd-keepalive/ka-agent/main.c
@@ -0,0 +1,150 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/wait.h>
+#include <sys/queue.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <time.h>
+
+#include <rte_keepalive.h>
+
+#include <shm.h>
+
+#define MAX_TIMEOUTS 4
+#define SEM_TIMEOUT_SECS 2
+
+static struct rte_keepalive_shm *ka_shm_create(void)
+{
+	int fd = shm_open(RTE_KEEPALIVE_SHM_NAME, O_RDWR, 0666);
+	size_t size = sizeof(struct rte_keepalive_shm);
+	struct rte_keepalive_shm *shm;
+
+	if (fd < 0)
+		printf("Failed to open %s as SHM:%s\n",
+			RTE_KEEPALIVE_SHM_NAME,
+		strerror(errno));
+	else {
+		shm = (struct rte_keepalive_shm *) mmap(
+			0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+		close(fd);
+		if (shm == MAP_FAILED)
+			printf("Failed to mmap SHM:%s\n", strerror(errno));
+		else
+			return shm;
+	}
+
+	/* Reset to zero, as it was set to MAP_FAILED aka: (void *)-1 */
+	shm = 0;
+	return NULL;
+}
+
+int main(void)
+{
+	struct rte_keepalive_shm *shm = ka_shm_create();
+	struct timespec timeout = { .tv_nsec = 0 };
+	int idx_core;
+	int cnt_cores;
+	uint64_t last_seen_alive_time = 0;
+	uint64_t most_recent_alive_time;
+	int cnt_timeouts = 0;
+	int sem_errno;
+
+	if (shm == NULL) {
+		printf("Unable to access shared core state\n");
+		return 1;
+	}
+	while (1) {
+		most_recent_alive_time = 0;
+		for (idx_core = 0; idx_core < RTE_KEEPALIVE_MAXCORES;
+				idx_core++)
+			if (shm->core_last_seen_times[idx_core] >
+					most_recent_alive_time)
+				most_recent_alive_time =
+					shm->core_last_seen_times[idx_core];
+
+		timeout.tv_sec = time(NULL) + SEM_TIMEOUT_SECS;
+		if (sem_timedwait(&shm->core_died, &timeout) == -1) {
+			/* Assume no core death signals and no change in any
+			 * last-seen times is the keepalive monitor itself
+			 * failing.
+			 */
+			sem_errno = errno;
+			last_seen_alive_time = most_recent_alive_time;
+			if (sem_errno == ETIMEDOUT) {
+				if (last_seen_alive_time ==
+						most_recent_alive_time &&
+						cnt_timeouts++ >
+						MAX_TIMEOUTS) {
+					printf("No updates. Exiting..\n");
+					break;
+					}
+			} else
+				printf("sem_timedwait() error (%s)\n",
+					strerror(sem_errno));
+			continue;
+		}
+		cnt_timeouts = 0;
+
+		cnt_cores = 0;
+		for (idx_core = 0; idx_core < RTE_KEEPALIVE_MAXCORES;
+				idx_core++)
+			if (shm->core_state[idx_core] == RTE_KA_STATE_DEAD)
+				cnt_cores++;
+		if (cnt_cores == 0) {
+			/* Can happen if core was restarted since Semaphore
+			 * was sent, due to agent being offline.
+			 */
+			printf("Warning: Empty dead core report\n");
+			continue;
+		}
+
+		printf("%i dead cores: ", cnt_cores);
+		for (idx_core = 0;
+				idx_core < RTE_KEEPALIVE_MAXCORES;
+				idx_core++)
+			if (shm->core_state[idx_core] == RTE_KA_STATE_DEAD)
+				printf("%d, ", idx_core);
+		printf("\b\b\n");
+	}
+	if (munmap(shm, sizeof(struct rte_keepalive_shm)) != 0)
+		printf("Warning: munmap() failed\n");
+	return 0;
+}
diff --git a/examples/l2fwd-keepalive/main.c b/examples/l2fwd-keepalive/main.c
index 4a0d9b6..84a59eb 100644
--- a/examples/l2fwd-keepalive/main.c
+++ b/examples/l2fwd-keepalive/main.c
@@ -72,6 +72,8 @@
 #include <rte_timer.h>
 #include <rte_keepalive.h>
 
+#include "shm.h"
+
 #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1
 
 #define NB_MBUF   8192
@@ -523,7 +525,7 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask)
 }
 
 static void
-dead_core(__attribute__((unused)) void *ptr_data, const int id_core)
+dead_core(__rte_unused void *ptr_data, const int id_core)
 {
 	printf("Dead core %i - restarting..\n", id_core);
 	if (rte_eal_get_lcore_state(id_core) == FINISHED) {
@@ -534,6 +536,14 @@ dead_core(__attribute__((unused)) void *ptr_data, const int id_core)
 	}
 }
 
+static void
+relay_core_state(void *ptr_data, const int id_core,
+	const enum rte_keepalive_state core_state, uint64_t last_alive)
+{
+	rte_keepalive_relayed_state((struct rte_keepalive_shm *)ptr_data,
+		id_core, core_state, last_alive);
+}
+
 int
 main(int argc, char **argv)
 {
@@ -722,10 +732,18 @@ main(int argc, char **argv)
 	rte_timer_init(&stats_timer);
 
 	if (check_period > 0) {
+		struct rte_keepalive_shm *ka_shm;
+
+		ka_shm = rte_keepalive_shm_create();
+		if (ka_shm == NULL)
+			rte_exit(EXIT_FAILURE,
+				"rte_keepalive_shm_create() failed");
 		rte_global_keepalive_info =
-			rte_keepalive_create(&dead_core, NULL);
+			rte_keepalive_create(&dead_core, ka_shm);
 		if (rte_global_keepalive_info == NULL)
 			rte_exit(EXIT_FAILURE, "init_keep_alive() failed");
+		rte_keepalive_register_relay_callback(rte_global_keepalive_info,
+			relay_core_state, ka_shm);
 		rte_timer_init(&hb_timer);
 		if (rte_timer_reset(&hb_timer,
 				(check_period * rte_get_timer_hz()) / 1000,
diff --git a/examples/l2fwd-keepalive/shm.c b/examples/l2fwd-keepalive/shm.c
new file mode 100644
index 0000000..66fc433
--- /dev/null
+++ b/examples/l2fwd-keepalive/shm.c
@@ -0,0 +1,129 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <time.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_keepalive.h>
+
+#include "shm.h"
+
+struct rte_keepalive_shm *rte_keepalive_shm_create(void)
+{
+	int fd;
+	int idx_core;
+	struct rte_keepalive_shm *ka_shm;
+
+	/* If any existing object is not unlinked, it makes it all too easy
+	 * for clients to end up with stale shared memory blocks when
+	 * restarted. Unlinking makes sure subsequent shm_open by clients
+	 * will get the new block mapped below.
+	 */
+	if (shm_unlink(RTE_KEEPALIVE_SHM_NAME) == -1 && errno != ENOENT)
+		printf("Warning: Error unlinking stale %s (%s)\n",
+			RTE_KEEPALIVE_SHM_NAME, strerror(errno));
+
+	fd = shm_open(RTE_KEEPALIVE_SHM_NAME,
+		O_CREAT | O_TRUNC | O_RDWR, 0666);
+	if (fd < 0)
+		RTE_LOG(INFO, EAL,
+			"Failed to open %s as SHM (%s)\n",
+			RTE_KEEPALIVE_SHM_NAME,
+			strerror(errno));
+	else if (ftruncate(fd, sizeof(struct rte_keepalive_shm)) != 0)
+		RTE_LOG(INFO, EAL,
+			"Failed to resize SHM (%s)\n", strerror(errno));
+	else {
+		ka_shm = (struct rte_keepalive_shm *) mmap(
+			0, sizeof(struct rte_keepalive_shm),
+			PROT_READ | PROT_WRITE,	MAP_SHARED, fd, 0);
+		close(fd);
+		if (ka_shm == MAP_FAILED)
+			RTE_LOG(INFO, EAL,
+				"Failed to mmap SHM (%s)\n", strerror(errno));
+		else {
+			memset(ka_shm, 0, sizeof(struct rte_keepalive_shm));
+
+			/* Initialize the semaphores for IPC/SHM use */
+			if (sem_init(&ka_shm->core_died, 1, 0) != 0) {
+				RTE_LOG(INFO, EAL,
+					"Failed to setup SHM semaphore (%s)\n",
+					strerror(errno));
+				return NULL;
+			}
+
+			/* Set all cores to 'not present' */
+			for (idx_core = 0;
+					idx_core < RTE_KEEPALIVE_MAXCORES;
+					idx_core++) {
+				ka_shm->core_state[idx_core] =
+					RTE_KA_STATE_UNUSED;
+				ka_shm->core_last_seen_times[idx_core] = 0;
+			}
+
+			return ka_shm;
+		}
+	}
+return NULL;
+}
+
+void rte_keepalive_relayed_state(struct rte_keepalive_shm *shm,
+	const int id_core, const enum rte_keepalive_state core_state,
+	__rte_unused uint64_t last_alive)
+{
+	int count;
+
+	shm->core_state[id_core] = core_state;
+	shm->core_last_seen_times[id_core] = last_alive;
+
+	if (core_state == RTE_KEEPALIVE_SHM_DEAD) {
+		/* Since core has died, also signal ka_agent.
+		 *
+		 * Limit number of times semaphore can be incremented, in case
+		 * ka_agent is not active.
+		 */
+		if (sem_getvalue(&shm->core_died, &count) == -1) {
+			RTE_LOG(INFO, EAL, "Semaphore check failed(%s)\n",
+				strerror(errno));
+			return;
+		}
+		if (count > 1)
+			return;
+
+		if (sem_post(&shm->core_died) != 0)
+			RTE_LOG(INFO, EAL,
+				"Failed to increment semaphore (%s)\n",
+				strerror(errno));
+	}
+}
diff --git a/examples/l2fwd-keepalive/shm.h b/examples/l2fwd-keepalive/shm.h
new file mode 100644
index 0000000..25e1b61
--- /dev/null
+++ b/examples/l2fwd-keepalive/shm.h
@@ -0,0 +1,89 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2016 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#define RTE_KEEPALIVE_SHM_NAME "/dpdk_keepalive_shm_name"
+
+#define RTE_KEEPALIVE_SHM_ALIVE 1
+#define RTE_KEEPALIVE_SHM_DEAD 2
+
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <semaphore.h>
+#include <rte_keepalive.h>
+
+/**
+ * Keepalive SHM structure.
+ *
+ * The shared memory allocated by the primary is this size, and contains the
+ * information as contained within this struct. A secondary may open the SHM,
+ * and read the contents.
+ */
+struct rte_keepalive_shm {
+	/** IPC semaphore. Posted when a core dies */
+	sem_t core_died;
+
+	/**
+	 * Relayed status of each core.
+	 */
+	enum rte_keepalive_state core_state[RTE_KEEPALIVE_MAXCORES];
+
+	/**
+	 * Last-seen-alive timestamps for the cores
+	 */
+	uint64_t core_last_seen_times[RTE_KEEPALIVE_MAXCORES];
+};
+
+/**
+ * Create shared host memory keepalive object.
+ * @return
+ *  Pointer to SHM keepalive structure, or NULL on failure.
+ */
+struct rte_keepalive_shm *rte_keepalive_shm_create(void);
+
+/**
+ * Relays state for given core
+ * @param *shm
+ *  Pointer to SHM keepalive structure.
+ * @param id_core
+ *  Id of core
+ * @param core_state
+ *  State of core
+ * @param last_alive
+ *  Last seen timestamp for core
+ */
+void rte_keepalive_relayed_state(struct rte_keepalive_shm *shm,
+	const int id_core, const enum rte_keepalive_state core_state,
+	uint64_t last_alive);
-- 
2.5.5

Re: [PATCH v4 0/3] Keep-alive enhancements

[Thomas Monjalon]

> Remy Horton (3):
>   eal: export keepalive state enumerations
>   eal: add additional keepalive callbacks
>   examples/l2fwd-keepalive: add IPC liveness reporting

Applied, thanks

Just a last comment: the agent in the example should not appear
in examples/Makefile.