[PATCH 1/6] service cores: header and implementation

[PATCH 1/6] service cores: header and implementation

[Harry van Haaren]

Add header files, update .map files with new service
functions, and add the service header to the doxygen
for building.

This service header API allows DPDK to use services as
a concept of something that requires CPU cycles. An example
is a PMD that runs in software to schedule events, where a
hardware version exists that does not require a CPU.

The code presented here is based on an initial RFC:
http://dpdk.org/ml/archives/dev/2017-May/065207.html

This was then reworked, and RFC v2 with the changes posted:
http://dpdk.org/ml/archives/dev/2017-June/067194.html

This is the third iteration of the service core concept,
now with an implementation.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
---
 doc/api/doxy-api-index.md                          |   1 +
 lib/librte_eal/bsdapp/eal/Makefile                 |   1 +
 lib/librte_eal/bsdapp/eal/rte_eal_version.map      |  24 +
 lib/librte_eal/common/Makefile                     |   1 +
 lib/librte_eal/common/include/rte_eal.h            |   4 +
 lib/librte_eal/common/include/rte_lcore.h          |   3 +-
 lib/librte_eal/common/include/rte_service.h        | 274 ++++++++++
 .../common/include/rte_service_private.h           | 108 ++++
 lib/librte_eal/common/rte_service.c                | 568 +++++++++++++++++++++
 lib/librte_eal/linuxapp/eal/Makefile               |   1 +
 lib/librte_eal/linuxapp/eal/rte_eal_version.map    |  24 +
 11 files changed, 1008 insertions(+), 1 deletion(-)
 create mode 100644 lib/librte_eal/common/include/rte_service.h
 create mode 100644 lib/librte_eal/common/include/rte_service_private.h
 create mode 100644 lib/librte_eal/common/rte_service.c

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index f5f1f19..55d522a 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -158,6 +158,7 @@ There are many libraries, so their headers may be grouped by topics:
   [common]             (@ref rte_common.h),
   [ABI compat]         (@ref rte_compat.h),
   [keepalive]          (@ref rte_keepalive.h),
+  [Service Cores]      (@ref rte_service.h),
   [device metrics]     (@ref rte_metrics.h),
   [bitrate statistics] (@ref rte_bitrate.h),
   [latency statistics] (@ref rte_latencystats.h),
diff --git a/lib/librte_eal/bsdapp/eal/Makefile b/lib/librte_eal/bsdapp/eal/Makefile
index a0f9950..05517a2 100644
--- a/lib/librte_eal/bsdapp/eal/Makefile
+++ b/lib/librte_eal/bsdapp/eal/Makefile
@@ -87,6 +87,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_malloc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_elem.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_heap.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_keepalive.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_service.c
 
 # from arch dir
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_cpuflags.c
diff --git a/lib/librte_eal/bsdapp/eal/rte_eal_version.map b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
index 2e48a73..843d4ee 100644
--- a/lib/librte_eal/bsdapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
@@ -193,3 +193,27 @@ DPDK_17.05 {
 	vfio_get_group_no;
 
 } DPDK_17.02;
+
+DPDK_17.08 {
+	global:
+
+	rte_service_core_add;
+	rte_service_core_count;
+	rte_service_core_del;
+	rte_service_core_list;
+	rte_service_core_reset_all;
+	rte_service_core_start;
+	rte_service_core_stop;
+	rte_service_disable_on_core;
+	rte_service_enable_on_core;
+	rte_service_get_by_id;
+	rte_service_get_count;
+	rte_service_get_enabled_on_core;
+	rte_service_is_running;
+	rte_service_register;
+	rte_service_reset;
+	rte_service_start;
+	rte_service_stop;
+	rte_service_unregister;
+
+} DPDK_17.05;
diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
index a5bd108..2a93397 100644
--- a/lib/librte_eal/common/Makefile
+++ b/lib/librte_eal/common/Makefile
@@ -41,6 +41,7 @@ INC += rte_eal_memconfig.h rte_malloc_heap.h
 INC += rte_hexdump.h rte_devargs.h rte_bus.h rte_dev.h rte_vdev.h
 INC += rte_pci_dev_feature_defs.h rte_pci_dev_features.h
 INC += rte_malloc.h rte_keepalive.h rte_time.h
+INC += rte_service.h rte_service_private.h
 
 GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h
 GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h
diff --git a/lib/librte_eal/common/include/rte_eal.h b/lib/librte_eal/common/include/rte_eal.h
index abf020b..1f203f8 100644
--- a/lib/librte_eal/common/include/rte_eal.h
+++ b/lib/librte_eal/common/include/rte_eal.h
@@ -61,6 +61,7 @@ extern "C" {
 enum rte_lcore_role_t {
 	ROLE_RTE,
 	ROLE_OFF,
+	ROLE_SERVICE,
 };
 
 /**
@@ -80,6 +81,7 @@ enum rte_proc_type_t {
 struct rte_config {
 	uint32_t master_lcore;       /**< Id of the master lcore */
 	uint32_t lcore_count;        /**< Number of available logical cores. */
+	uint32_t score_count;        /**< Number of available service cores. */
 	enum rte_lcore_role_t lcore_role[RTE_MAX_LCORE]; /**< State of cores. */
 
 	/** Primary or secondary configuration */
@@ -185,6 +187,8 @@ int rte_eal_iopl_init(void);
  *
  *     EPROTO indicates that the PCI bus is either not present, or is not
  *            readable by the eal.
+ *
+ *     ENOEXEC indicates that a service core failed to launch successfully.
  */
 int rte_eal_init(int argc, char **argv);
 
diff --git a/lib/librte_eal/common/include/rte_lcore.h b/lib/librte_eal/common/include/rte_lcore.h
index fe7b586..50e0d0f 100644
--- a/lib/librte_eal/common/include/rte_lcore.h
+++ b/lib/librte_eal/common/include/rte_lcore.h
@@ -73,6 +73,7 @@ struct lcore_config {
 	unsigned core_id;          /**< core number on socket for this lcore */
 	int core_index;            /**< relative index, starting from 0 */
 	rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
+	uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
 };
 
 /**
@@ -175,7 +176,7 @@ rte_lcore_is_enabled(unsigned lcore_id)
 	struct rte_config *cfg = rte_eal_get_configuration();
 	if (lcore_id >= RTE_MAX_LCORE)
 		return 0;
-	return cfg->lcore_role[lcore_id] != ROLE_OFF;
+	return cfg->lcore_role[lcore_id] == ROLE_RTE;
 }
 
 /**
diff --git a/lib/librte_eal/common/include/rte_service.h b/lib/librte_eal/common/include/rte_service.h
new file mode 100644
index 0000000..de079dd
--- /dev/null
+++ b/lib/librte_eal/common/include/rte_service.h
@@ -0,0 +1,274 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_SERVICE_H_
+#define _RTE_SERVICE_H_
+
+/**
+ * @file
+ *
+ * Service functions
+ *
+ * The service functionality provided by this header allows a DPDK component
+ * to indicate that it requires a function call in order for it to perform
+ * its processing.
+ *
+ * An example usage of this functionality would be a component that registers
+ * a service to perform a particular packet processing duty: for example the
+ * eventdev software PMD. At startup the application requests all services
+ * that have been registered, and the cores in the service-coremask run the
+ * required services. The EAL removes these number of cores from the available
+ * runtime cores, and dedicates them to performing service-core workloads. The
+ * application has access to the remaining lcores as normal.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include<stdio.h>
+#include <stdint.h>
+#include <sys/queue.h>
+
+#include <rte_lcore.h>
+
+/* forward declaration only. Definition in rte_service_private.h */
+struct rte_service_spec;
+
+#define RTE_SERVICE_NAME_MAX 32
+
+/* Capabilities of a service.
+ *
+ * Use the *rte_service_probe_capability* function to check if a service is
+ * capable of a specific capability.
+ */
+/** When set, the service is capable of having multiple threads run it at the
+ *  same time.
+ */
+#define RTE_SERVICE_CAP_MT_SAFE (1 << 0)
+
+/** Return the number of services registered.
+ *
+ * The number of services registered can be passed to *rte_service_get_by_id*,
+ * enabling the application to retireve the specificaion of each service.
+ *
+ * @return The number of services registered.
+ */
+uint32_t rte_service_get_count(void);
+
+/** Return the specificaion of each service.
+ *
+ * This function provides the specification of a service. This can be used by
+ * the application to understand what the service represents. The service
+ * must not be modified by the application directly, only passed to the various
+ * rte_service_* functions.
+ *
+ * @param id The integer id of the service to retrieve
+ * @retval non-zero A valid pointer to the service_spec
+ * @retval NULL Invalid *id* provided.
+ */
+struct rte_service_spec *rte_service_get_by_id(uint32_t id);
+
+/** Return the name of the service.
+ *
+ * @return A pointer to the name of the service. The returned pointer remains
+ *         in ownership of the service, and the application must not free it.
+ */
+const char *rte_service_get_name(const struct rte_service_spec *service);
+
+/* Check if a service has a specific capability.
+ *
+ * This function returns if *service* has implements *capability*.
+ * See RTE_SERVICE_CAP_* defines for a list of valid capabilities.
+ * @retval 1 Capability supported by this service instance
+ * @retval 0 Capability not supported by this service instance
+ */
+int32_t rte_service_probe_capability(const struct rte_service_spec *service,
+				     uint32_t capability);
+
+/** Enable a core to run a service.
+ *
+ * Each core can be added or removed from running specific services. This
+ * functions adds *lcore* to the set of cores that will run *service*.
+ *
+ * If multiple cores are enabled on a service, an atomic is used to ensure that
+ * only one cores runs the service at a time. The exception to this is when
+ * a service indicates that it is multi-thread safe by setting the capability
+ * called RTE_SERVICE_CAP_MT_SAFE. With the multi-thread safe capability set,
+ * the service function can be run on multiple threads at the same time.
+ *
+ * @retval 0 lcore added successfully
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_enable_on_core(struct rte_service_spec *service,
+				   uint32_t lcore);
+
+/** Disable a core to run a service.
+ *
+ * Each core can be added or removed from running specific services. This
+ * functions removes *lcore* to the set of cores that will run *service*.
+ *
+ * @retval 0 Lcore removed successfully
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_disable_on_core(struct rte_service_spec *service,
+				   uint32_t lcore);
+
+/** Return if an lcore is enabled for the service.
+ *
+ * This function allows the application to query if *lcore* is currently set to
+ * run *service*.
+ *
+ * @retval 1 Lcore enabled on this lcore
+ * @retval 0 Lcore disabled on this lcore
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_get_enabled_on_core(struct rte_service_spec *service,
+					uint32_t lcore);
+
+
+/** Enable *service* to run.
+ *
+ * This function switches on a service during runtime.
+ * @retval 0 The service was successfully started
+ */
+int32_t rte_service_start(struct rte_service_spec *service);
+
+/** Disable *service*.
+ *
+ * Switch off a service, so it is not run until it is *rte_service_start* is
+ * called on it.
+ * @retval 0 Service successfully switched off
+ */
+int32_t rte_service_stop(struct rte_service_spec *service);
+
+/** Returns if *service* is currently running.
+ *
+ * @retval 1 Service is currently running
+ * @retval 0 Service is currently stopped
+ * @retval -EINVAL Invalid service pointer provided
+ */
+int32_t rte_service_is_running(const struct rte_service_spec *service);
+
+/* Start a service core.
+ *
+ * Starting a core makes the core begin polling. Any services assigned to it
+ * will be run as fast as possible.
+ *
+ * @retval 0 Success
+ * @retval -EINVAL Failed to start core. The *lcore_id* passed in is not
+ *          currently assigned to be a service core.
+ */
+int32_t rte_service_core_start(uint32_t lcore_id);
+
+/* Stop a service core.
+ *
+ * Stopping a core makes the core become idle, but remains  assigned as a
+ * service core.
+ *
+ * @retval 0 Success
+ * @retval -EINVAL Invalid *lcore_id* provided
+ * @retval -EALREADY Already stopped core
+ * @retval -EBUSY Failed to stop core, as it would cause a service to not
+ *          be run, as this is the only core currently running the service.
+ *          The application must stop the service first, and then stop the
+ *          lcore.
+ */
+int32_t rte_service_core_stop(uint32_t lcore_id);
+
+/** Adds lcore to the list of service cores.
+ *
+ * This functions can be used at runtime in order to modify the service core
+ * mask.
+ *
+ * @retval 0 Success
+ * @retval -EBUSY lcore is busy, and not available for service core duty
+ * @retval -EALREADY lcore is already added to the service core list
+ * @retval -EINVAL Invalid lcore provided
+ */
+int32_t rte_service_core_add(uint32_t lcore);
+
+/** Removes lcore from the list of service cores.
+ *
+ * This can fail if the core is not stopped, see *rte_service_core_stop*.
+ *
+ * @retval 0 Success
+ * @retval -EBUSY Lcore is not stopped, stop service core before removing.
+ * @retval -EINVAL failed to add lcore to service core mask.
+ */
+int32_t rte_service_core_del(uint32_t lcore);
+
+/** Retreve the number of service cores currently avaialble.
+ *
+ * This function returns the integer count of service cores available. The
+ * service core count can be used in mapping logic when creating mappings
+ * from service cores to services.
+ *
+ * See *rte_service_core_list* for details on retrieving the lcore_id of each
+ * service core.
+ *
+ * @return The number of service cores currently configured.
+ */
+int32_t rte_service_core_count(void);
+
+/** Reset all service core mappings.
+ * @retval 0 Success
+ */
+int32_t rte_service_core_reset_all(void);
+
+/** Retrieve the list of currently enabled service cores.
+ *
+ * This function fills in an application supplied array, with each element
+ * indicating the lcore_id of a service core.
+ *
+ * Adding and removing service cores can be performed using
+ * *rte_service_core_add* and *rte_service_core_del*.
+ * @param array An array of at least N items.
+ * @param The size of *array*.
+ * @retval >=0 Number of service cores that have been populated in the array
+ * @retval -ENOMEM The provided array is not large enough to fill in the
+ *          service core list. No items have been populated, call this function
+ *          with a size of at least *rte_service_core_count* items.
+ */
+int32_t rte_service_core_list(uint32_t array[], uint32_t n);
+
+/** Dumps any information available about the service. If service is NULL,
+ * dumps info for all services.
+ */
+int32_t rte_service_dump(FILE *f, struct rte_service_spec *service);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* _RTE_SERVICE_H_ */
diff --git a/lib/librte_eal/common/include/rte_service_private.h b/lib/librte_eal/common/include/rte_service_private.h
new file mode 100644
index 0000000..d8bb644
--- /dev/null
+++ b/lib/librte_eal/common/include/rte_service_private.h
@@ -0,0 +1,108 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_SERVICE_PRIVATE_H_
+#define _RTE_SERVICE_PRIVATE_H_
+
+/* This file specifies the internal service specification.
+ * Include this file if you are writing a component that requires CPU cycles to
+ * operate, and you wish to run the component using service cores
+ */
+
+#include <rte_service.h>
+
+/**
+ * Signature of callback function to run a service.
+ */
+typedef int32_t (*rte_service_func)(void *args);
+
+/**
+ * The specification of a service.
+ *
+ * This struct contains metadata about the service itself, the callback
+ * function to run one iteration of the service, a userdata pointer, flags etc.
+ */
+struct rte_service_spec {
+	/** The name of the service. This should be used by the application to
+	 * understand what purpose this service provides.
+	 */
+	char name[RTE_SERVICE_NAME_MAX];
+	/** The callback to invoke to run one iteration of the service. */
+	rte_service_func callback;
+	/** The userdata pointer provided to the service callback. */
+	void *callback_userdata;
+	/** Flags to indicate the capabilities of this service. See defines in
+	 * the public header file for values of RTE_SERVICE_CAP_*
+	 */
+	uint32_t capabilities;
+	/** NUMA socket ID that this service is affinitized to */
+	int8_t socket_id;
+};
+
+/** Register a new service.
+ *
+ * A service represents a component that the requires CPU time periodically to
+ * achieve its purpose.
+ *
+ * For example the eventdev SW PMD requires CPU cycles to perform its
+ * scheduling. This can be achieved by registering it as a service, and the
+ * application can then assign CPU resources to it using
+ * *rte_service_set_coremask*.
+ *
+ * @param spec The specification of the service to register
+ * @retval 0 Successfully registered the service.
+ *         -EINVAL Attempted to register an invalid service (eg, no callback
+ *         set)
+ */
+int32_t rte_service_register(const struct rte_service_spec *spec);
+
+/** Unregister a service.
+ *
+ * The service being removed must be stopped before calling this function.
+ *
+ * @retval 0 The service was successfully unregistered.
+ * @retval -EBUSY The service is currently running, stop the service before
+ *          calling unregister. No action has been taken.
+ */
+int32_t rte_service_unregister(struct rte_service_spec *service);
+
+/** Private function to allow EAL to initialied default mappings.
+ *
+ * This function iterates all the services, and maps then to the available
+ * cores. Based on the capabilities of the services, they are set to run on the
+ * available cores in a round-robin manner.
+ *
+ * @retval 0 Success
+ */
+int32_t rte_service_init_default_mapping(void);
+
+#endif /* _RTE_SERVICE_PRIVATE_H_ */
diff --git a/lib/librte_eal/common/rte_service.c b/lib/librte_eal/common/rte_service.c
new file mode 100644
index 0000000..8b5e344
--- /dev/null
+++ b/lib/librte_eal/common/rte_service.c
@@ -0,0 +1,568 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_service.h>
+#include "include/rte_service_private.h"
+
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_common.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_atomic.h>
+
+#define RTE_SERVICE_NUM_MAX 64
+
+#define RTE_SERVICE_FLAG_REGISTERED_SHIFT 0
+
+#define RTE_SERVICE_RUNSTATE_STOPPED 0
+#define RTE_SERVICE_RUNSTATE_RUNNING 1
+
+/* internal representation of a service */
+struct rte_service_spec_impl {
+	/* public part of the struct */
+	struct rte_service_spec spec;
+
+	/* atomic lock that when set indicates a service core is currently
+	 * running this service callback. When not set, a core may take the
+	 * lock and then run the service callback.
+	 */
+	rte_atomic32_t execute_lock;
+
+	/* API set/get-able variables */
+	int32_t runstate;
+	uint8_t internal_flags;
+
+	/* per service statistics */
+	uint32_t num_mapped_cores;
+	uint64_t calls;
+	uint64_t cycles_spent;
+};
+
+/* the internal values of a service core */
+struct core_state {
+	uint64_t service_mask; /* map of services IDs are run on this core */
+	uint8_t runstate; /* running or stopped */
+	uint8_t is_service_core; /* set if core is currently a service core */
+
+	/* extreme statistics */
+	uint64_t calls_per_service[RTE_SERVICE_NUM_MAX];
+};
+
+static uint32_t rte_service_count;
+static struct rte_service_spec_impl rte_services[RTE_SERVICE_NUM_MAX];
+static struct core_state cores_state[RTE_MAX_LCORE];
+
+/* returns 1 if service is registered and has not been unregistered
+ * Returns 0 if service never registered, or has been unregistered
+ */
+static int
+service_valid(uint32_t id) {
+	return !!(rte_services[id].internal_flags &
+		 (1 << RTE_SERVICE_FLAG_REGISTERED_SHIFT));
+}
+
+uint32_t
+rte_service_get_count(void)
+{
+	return rte_service_count;
+}
+
+struct rte_service_spec *
+rte_service_get_by_id(uint32_t id)
+{
+	struct rte_service_spec *service = NULL;
+	if (id < rte_service_count)
+		service = (struct rte_service_spec *)&rte_services[id];
+
+	return service;
+}
+
+const char *
+rte_service_get_name(const struct rte_service_spec *service)
+{
+	return service->name;
+}
+
+int32_t
+rte_service_probe_capability(const struct rte_service_spec *service,
+			     uint32_t capability)
+{
+	return service->capabilities & capability;
+}
+
+int32_t
+rte_service_is_running(const struct rte_service_spec *spec)
+{
+	if (!spec)
+		return -EINVAL;
+
+	const struct rte_service_spec_impl *impl =
+		(const struct rte_service_spec_impl *)spec;
+	return impl->runstate == RTE_SERVICE_RUNSTATE_RUNNING;
+}
+
+int32_t
+rte_service_register(const struct rte_service_spec *spec)
+{
+	uint32_t i;
+	int32_t free_slot = -1;
+
+	if (spec->callback == NULL || strlen(spec->name) == 0)
+		return -EINVAL;
+
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (!service_valid(i)) {
+			free_slot = i;
+			break;
+		}
+	}
+
+	if (free_slot < 0)
+		return -ENOSPC;
+
+	struct rte_service_spec_impl *s = &rte_services[free_slot];
+	s->spec = *spec;
+	s->internal_flags |= (1 << RTE_SERVICE_FLAG_REGISTERED_SHIFT);
+
+	rte_smp_wmb();
+	rte_service_count++;
+
+	return 0;
+}
+
+int32_t
+rte_service_unregister(struct rte_service_spec *spec)
+{
+	struct rte_service_spec_impl *s = NULL;
+	struct rte_service_spec_impl *spec_impl =
+		(struct rte_service_spec_impl *)spec;
+
+	uint32_t i;
+	uint32_t service_id;
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (&rte_services[i] == spec_impl) {
+			s = spec_impl;
+			service_id = i;
+			break;
+		}
+	}
+
+	if (!s)
+		return -EINVAL;
+
+	s->internal_flags &= ~(1 << RTE_SERVICE_FLAG_REGISTERED_SHIFT);
+
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		cores_state[i].service_mask &= ~(1 << service_id);
+
+	memset(&rte_services[service_id], 0,
+			sizeof(struct rte_service_spec_impl));
+
+	rte_smp_wmb();
+	rte_service_count--;
+
+	return 0;
+}
+
+int32_t
+rte_service_start(struct rte_service_spec *service)
+{
+	struct rte_service_spec_impl *s =
+		(struct rte_service_spec_impl *)service;
+	s->runstate = RTE_SERVICE_RUNSTATE_RUNNING;
+	return 0;
+}
+
+int32_t
+rte_service_stop(struct rte_service_spec *service)
+{
+	struct rte_service_spec_impl *s =
+		(struct rte_service_spec_impl *)service;
+	s->runstate = RTE_SERVICE_RUNSTATE_STOPPED;
+	return 0;
+}
+
+static int32_t
+rte_service_runner_func(void *arg)
+{
+	RTE_SET_USED(arg);
+	uint32_t i;
+	const int lcore = rte_lcore_id();
+	struct core_state *cs = &cores_state[lcore];
+
+	while (cores_state[lcore].runstate == RTE_SERVICE_RUNSTATE_RUNNING) {
+		for (i = 0; i < rte_service_count; i++) {
+			struct rte_service_spec_impl *s = &rte_services[i];
+			uint64_t service_mask = cs->service_mask;
+
+			if (s->runstate != RTE_SERVICE_RUNSTATE_RUNNING ||
+					!(service_mask & (1 << i)))
+				continue;
+
+			uint32_t *lock = (uint32_t *)&s->execute_lock;
+			if (rte_atomic32_cmpset(lock, 0, 1)) {
+				void *userdata = s->spec.callback_userdata;
+				uint64_t start = rte_rdtsc();
+				s->spec.callback(userdata);
+				uint64_t end = rte_rdtsc();
+
+				uint64_t spent = end - start;
+				s->cycles_spent += spent;
+				s->calls++;
+				cs->calls_per_service[i]++;
+
+				rte_atomic32_clear(&s->execute_lock);
+			}
+		}
+		rte_mb();
+	}
+
+	/* mark core as ready to accept work again */
+	lcore_config[lcore].state = WAIT;
+
+	return 0;
+}
+
+int32_t
+rte_service_core_count(void)
+{
+	int32_t count = 0;
+	uint32_t i;
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		count += cores_state[i].is_service_core;
+	return count;
+}
+
+int32_t
+rte_service_core_list(uint32_t array[], uint32_t n)
+{
+	uint32_t count = rte_service_core_count();
+	if (count > n)
+		return -ENOMEM;
+
+	uint32_t i;
+	uint32_t idx = 0;
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct core_state *cs = &cores_state[i];
+		if (cs->is_service_core) {
+			array[idx] = i;
+			idx++;
+		}
+	}
+
+	return count;
+}
+
+int32_t
+rte_service_init_default_mapping(void)
+{
+	/* create a default mapping from cores to services, then start the
+	 * services to make them transparent to unaware applications.
+	 */
+	uint32_t i;
+	int ret;
+	uint32_t count = rte_service_get_count();
+	struct rte_config *cfg = rte_eal_get_configuration();
+
+	for (i = 0; i < count; i++) {
+		struct rte_service_spec *s = rte_service_get_by_id(i);
+		if (!s)
+			return -EINVAL;
+
+		ret = 0;
+		int j;
+		for (j = 0; j < RTE_MAX_LCORE; j++) {
+			/* TODO: add lcore -> service mapping logic here */
+			if (cfg->lcore_role[j] == ROLE_SERVICE) {
+				ret = rte_service_enable_on_core(s, j);
+				if (ret)
+					rte_panic("Enabling service core %d on service %s failed\n",
+							j, s->name);
+			}
+		}
+
+		ret = rte_service_start(s);
+		if (ret)
+			rte_panic("failed to start service %s\n", s->name);
+	}
+
+	return 0;
+}
+
+static int32_t
+service_update(struct rte_service_spec *service, uint32_t lcore,
+		uint32_t *set, uint32_t *enabled)
+{
+	uint32_t i;
+	int32_t sid = -1;
+
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if ((struct rte_service_spec *)&rte_services[i] == service &&
+				service_valid(i)) {
+			sid = i;
+			break;
+		}
+	}
+
+	if (sid == -1 || lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	if (!cores_state[lcore].is_service_core)
+		return -EINVAL;
+
+	if (set) {
+		if (*set) {
+			cores_state[lcore].service_mask |=  (1 << sid);
+			rte_services[sid].num_mapped_cores++;
+		} else {
+			cores_state[lcore].service_mask &= ~(1 << sid);
+			rte_services[sid].num_mapped_cores--;
+		}
+	}
+
+	if (enabled)
+		*enabled = (cores_state[lcore].service_mask & (1 << sid));
+
+	return 0;
+}
+
+int32_t rte_service_get_enabled_on_core(struct rte_service_spec *service,
+					uint32_t lcore)
+{
+	uint32_t enabled;
+	int ret = service_update(service, lcore, 0, &enabled);
+	if (ret == 0)
+		return enabled;
+	return -EINVAL;
+}
+
+int32_t
+rte_service_enable_on_core(struct rte_service_spec *service, uint32_t lcore)
+{
+	uint32_t on = 1;
+	return service_update(service, lcore, &on, 0);
+}
+
+int32_t
+rte_service_disable_on_core(struct rte_service_spec *service, uint32_t lcore)
+{
+	uint32_t off = 0;
+	return service_update(service, lcore, &off, 0);
+}
+
+int32_t rte_service_core_reset_all(void)
+{
+	/* loop over cores, reset all to mask 0 */
+	uint32_t i;
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		cores_state[i].service_mask = 0;
+		cores_state[i].is_service_core = 0;
+	}
+
+	return 0;
+}
+
+int32_t
+rte_service_core_add(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+	if (cores_state[lcore].is_service_core)
+		return -EALREADY;
+
+	lcore_config[lcore].core_role = ROLE_SERVICE;
+
+	/* TODO: take from EAL by setting ROLE_SERVICE? */
+	cores_state[lcore].is_service_core = 1;
+	cores_state[lcore].service_mask = 0;
+
+	return 0;
+}
+
+int32_t
+rte_service_core_del(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	struct core_state *cs = &cores_state[lcore];
+	if (!cs->is_service_core)
+		return -EINVAL;
+
+	if (cs->runstate != RTE_SERVICE_RUNSTATE_STOPPED)
+		return -EBUSY;
+
+	lcore_config[lcore].core_role = ROLE_RTE;
+	cores_state[lcore].is_service_core = 0;
+	/* TODO: return to EAL by setting ROLE_RTE? */
+
+	return 0;
+}
+
+int32_t
+rte_service_core_start(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	struct core_state *cs = &cores_state[lcore];
+	if (!cs->is_service_core)
+		return -EINVAL;
+
+	if (cs->runstate == RTE_SERVICE_RUNSTATE_RUNNING)
+		return -EALREADY;
+
+	/* set core to run state first, and then launch otherwise it will
+	 * return immidiatly as runstate keeps it in the service poll loop
+	 */
+	cores_state[lcore].runstate = RTE_SERVICE_RUNSTATE_RUNNING;
+
+	int ret = rte_eal_remote_launch(rte_service_runner_func, 0, lcore);
+	/* returns -EBUSY if the core is already launched, 0 on success */
+	return ret;
+}
+
+int32_t
+rte_service_core_stop(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	if (cores_state[lcore].runstate == RTE_SERVICE_RUNSTATE_STOPPED)
+		return -EALREADY;
+
+	uint32_t i;
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		int32_t enabled = cores_state[i].service_mask & (1 << i);
+		int32_t service_running = rte_services[i].runstate !=
+						RTE_SERVICE_RUNSTATE_STOPPED;
+		int32_t only_core = rte_services[i].num_mapped_cores == 1;
+
+		/* if the core is mapped, and the service is running, and this
+		 * is the only core that is mapped, the service would cease to
+		 * run if this core stopped, so fail instead.
+		 */
+		if (enabled && service_running && only_core)
+			return -EBUSY;
+	}
+
+	cores_state[lcore].runstate = RTE_SERVICE_RUNSTATE_STOPPED;
+
+	return 0;
+}
+
+static void
+rte_service_dump_one(FILE *f, struct rte_service_spec_impl *s,
+		     uint64_t all_cycles, uint32_t reset)
+{
+	/* avoid divide by zeros */
+	if (all_cycles == 0)
+		all_cycles = 1;
+
+	int calls = 1;
+	if (s->calls != 0)
+		calls = s->calls;
+
+	float cycles_pct = (((float)s->cycles_spent) / all_cycles) * 100.f;
+	fprintf(f,
+			"  %s : %0.1f %%\tcalls %"PRIu64"\tcycles %"PRIu64"\tavg: %"PRIu64"\n",
+			s->spec.name, cycles_pct, s->calls, s->cycles_spent,
+			s->cycles_spent / calls);
+
+	if (reset) {
+		s->cycles_spent = 0;
+		s->calls = 0;
+	}
+}
+
+static void
+service_dump_calls_per_core(FILE *f, uint32_t lcore, uint32_t reset)
+{
+	uint32_t i;
+	struct core_state *cs = &cores_state[lcore];
+
+	fprintf(f, "%02d\t", lcore);
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (!service_valid(i))
+			continue;
+		fprintf(f, "%04"PRIu64"\t", cs->calls_per_service[i]);
+		if (reset)
+			cs->calls_per_service[i] = 0;
+	}
+	fprintf(f, "\n");
+}
+
+int32_t rte_service_dump(FILE *f, struct rte_service_spec *service)
+{
+	uint32_t i;
+
+	uint64_t total_cycles = 0;
+	for (i = 0; i < rte_service_count; i++)
+		total_cycles += rte_services[i].cycles_spent;
+
+	if (service) {
+		struct rte_service_spec_impl *s =
+			(struct rte_service_spec_impl *)service;
+		fprintf(f, "Service %s Summary\n", s->spec.name);
+		uint32_t reset = 0;
+		rte_service_dump_one(f, s, total_cycles, reset);
+		return 0;
+	}
+
+	struct rte_config *cfg = rte_eal_get_configuration();
+
+	fprintf(f, "Services Summary\n");
+	for (i = 0; i < rte_service_count; i++) {
+		uint32_t reset = 1;
+		rte_service_dump_one(f, &rte_services[i], total_cycles, reset);
+	}
+
+	fprintf(f, "Service Cores Summary\n");
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		if (cfg->lcore_role[i] != ROLE_SERVICE)
+			continue;
+
+		uint32_t reset = 0;
+		service_dump_calls_per_core(f, i, reset);
+	}
+
+	return 0;
+}
diff --git a/lib/librte_eal/linuxapp/eal/Makefile b/lib/librte_eal/linuxapp/eal/Makefile
index 640afd0..438dcf9 100644
--- a/lib/librte_eal/linuxapp/eal/Makefile
+++ b/lib/librte_eal/linuxapp/eal/Makefile
@@ -96,6 +96,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_malloc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_elem.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_heap.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_keepalive.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_service.c
 
 # from arch dir
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_cpuflags.c
diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
index 670bab3..f0ed607 100644
--- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
@@ -198,3 +198,27 @@ DPDK_17.05 {
 	vfio_get_group_no;
 
 } DPDK_17.02;
+
+DPDK_17.08 {
+	global:
+
+	rte_service_core_add;
+	rte_service_core_count;
+	rte_service_core_del;
+	rte_service_core_list;
+	rte_service_core_reset_all;
+	rte_service_core_start;
+	rte_service_core_stop;
+	rte_service_disable_on_core;
+	rte_service_enable_on_core;
+	rte_service_get_by_id;
+	rte_service_get_count;
+	rte_service_get_enabled_on_core;
+	rte_service_is_running;
+	rte_service_register;
+	rte_service_reset;
+	rte_service_start;
+	rte_service_stop;
+	rte_service_unregister;
+
+} DPDK_17.05;
-- 
2.7.4

[PATCH 2/6] service cores: coremask parsing

[Harry van Haaren]

Add logic for parsing a coremask from EAL, which allows
the application to be unaware of the cores being taken from
its coremask.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
---
 lib/librte_eal/common/eal_common_options.c | 78 ++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)

diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index f470195..3599784 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -50,6 +50,7 @@
 #include <rte_version.h>
 #include <rte_devargs.h>
 #include <rte_memcpy.h>
+#include <rte_service.h>
 
 #include "eal_internal_cfg.h"
 #include "eal_options.h"
@@ -61,6 +62,7 @@ const char
 eal_short_options[] =
 	"b:" /* pci-blacklist */
 	"c:" /* coremask */
+	"s:" /* service coremask */
 	"d:" /* driver */
 	"h"  /* help */
 	"l:" /* corelist */
@@ -267,6 +269,75 @@ static int xdigit2val(unsigned char c)
 }
 
 static int
+eal_parse_service_coremask(const char *coremask)
+{
+	struct rte_config *cfg = rte_eal_get_configuration();
+	int i, j, idx = 0;
+	unsigned count = 0;
+	char c;
+	int val;
+
+	if (coremask == NULL)
+		return -1;
+	/* Remove all blank characters ahead and after .
+	 * Remove 0x/0X if exists.
+	 */
+	while (isblank(*coremask))
+		coremask++;
+	if (coremask[0] == '0' && ((coremask[1] == 'x')
+		|| (coremask[1] == 'X')))
+		coremask += 2;
+	i = strlen(coremask);
+	while ((i > 0) && isblank(coremask[i - 1]))
+		i--;
+
+	if (i == 0)
+		return -1;
+
+	printf("\n\nRemoving Service Cores from lcore roles now\n\n");
+
+	/* TODO: only scan active cores in coremask */
+	for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
+		c = coremask[i];
+		if (isxdigit(c) == 0) {
+			/* invalid characters */
+			return -1;
+		}
+		val = xdigit2val(c);
+		for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE;
+				j++, idx++) {
+			if ((1 << j) & val) {
+				/* TODO: enable flexible master core */
+				if (idx == 0)
+					continue;
+
+				if (!lcore_config[idx].detected) {
+					RTE_LOG(ERR, EAL,
+						"lcore %u unavailable\n", idx);
+					return -1;
+				}
+				//cfg->lcore_role[idx] = ROLE_SERVICE;
+				rte_service_core_add(idx);
+				count++;
+			}
+		}
+	}
+
+	for (; i >= 0; i--)
+		if (coremask[i] != '0')
+			return -1;
+
+	for (; idx < RTE_MAX_LCORE; idx++)
+		lcore_config[idx].core_index = -1;
+
+	if (count == 0)
+		return -1;
+
+	cfg->score_count = count;
+	return 0;
+}
+
+static int
 eal_parse_coremask(const char *coremask)
 {
 	struct rte_config *cfg = rte_eal_get_configuration();
@@ -826,6 +897,13 @@ eal_parse_common_option(int opt, const char *optarg,
 		}
 		core_parsed = 1;
 		break;
+	/* service coremask */
+	case 's':
+		if (eal_parse_service_coremask(optarg) < 0) {
+			RTE_LOG(ERR, EAL, "invalid service coremask\n");
+			return -1;
+		}
+		break;
 	/* size of memory */
 	case 'm':
 		conf->memory = atoi(optarg);
-- 
2.7.4

[PATCH 3/6] service cores: EAL init changes

[Harry van Haaren]

This commit shows the changes required in rte_eal_init()
to transparently launch the service threads. The threads
are launched into the service worker functions here because
after rte_eal_init() the application is not gauranteed to
call any other DPDK API.

As the registration of services happens at initialization
time, the services that require CPU time are already available
when we reach the end of rte_eal_init().

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
---
 lib/librte_eal/linuxapp/eal/eal.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
index 7c78f2d..4d6ad0e 100644
--- a/lib/librte_eal/linuxapp/eal/eal.c
+++ b/lib/librte_eal/linuxapp/eal/eal.c
@@ -78,6 +78,7 @@
 #include <rte_version.h>
 #include <rte_atomic.h>
 #include <malloc_heap.h>
+#include <rte_service_private.h>
 
 #include "eal_private.h"
 #include "eal_thread.h"
@@ -939,6 +940,20 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 
+	/* initialize service core threads and default service-core mapping */
+	struct rte_config *config = rte_eal_get_configuration();
+	uint32_t service_cores[RTE_MAX_LCORE];
+	int count = rte_service_core_list(service_cores, RTE_MAX_LCORE);
+	for (i = 0; i < count; i++) {
+		config->lcore_role[service_cores[i]] = ROLE_SERVICE;
+		rte_service_core_start(service_cores[i]);
+	}
+	ret = rte_service_init_default_mapping();
+	if (ret) {
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	rte_eal_mcfg_complete();
 
 	return fctret;
-- 
2.7.4

[PATCH 4/6] service cores: mark cores in lcore config as RTE

[Harry van Haaren]

Mark these cores as RTE available by default

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
---
 lib/librte_eal/common/eal_common_lcore.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/librte_eal/common/eal_common_lcore.c b/lib/librte_eal/common/eal_common_lcore.c
index 84fa0cb..0db1555 100644
--- a/lib/librte_eal/common/eal_common_lcore.c
+++ b/lib/librte_eal/common/eal_common_lcore.c
@@ -81,6 +81,7 @@ rte_eal_cpu_init(void)
 
 		/* By default, each detected core is enabled */
 		config->lcore_role[lcore_id] = ROLE_RTE;
+		lcore_config[lcore_id].core_role = ROLE_RTE;
 		lcore_config[lcore_id].core_id = eal_cpu_core_id(lcore_id);
 		lcore_config[lcore_id].socket_id = eal_cpu_socket_id(lcore_id);
 		if (lcore_config[lcore_id].socket_id >= RTE_MAX_NUMA_NODES) {
-- 
2.7.4

[PATCH 5/6] service core: add unit tests

[Harry van Haaren]

Add a bunch of unit tests, to ensure that the service
core functions are operating as expected.

As part of these tests a dummy service is registered which
allows identifying if a service callback has been invoked
by using the CPU tick counter. This allows identifying if
functions to start and stop service lcores are actually having
effect.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
---
 test/test/Makefile             |   2 +
 test/test/test_service_cores.c | 311 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 313 insertions(+)
 create mode 100644 test/test/test_service_cores.c

diff --git a/test/test/Makefile b/test/test/Makefile
index ee240be..61e296b 100644
--- a/test/test/Makefile
+++ b/test/test/Makefile
@@ -151,6 +151,8 @@ SRCS-y += test_interrupts.c
 SRCS-y += test_version.c
 SRCS-y += test_func_reentrancy.c
 
+SRCS-y += test_service_cores.c
+
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline.c
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_num.c
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_etheraddr.c
diff --git a/test/test/test_service_cores.c b/test/test/test_service_cores.c
new file mode 100644
index 0000000..299010a
--- /dev/null
+++ b/test/test/test_service_cores.c
@@ -0,0 +1,311 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <rte_common.h>
+#include <rte_hexdump.h>
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_memcpy.h>
+#include <rte_cycles.h>
+
+#include <rte_service.h>
+#include <rte_service_private.h>
+
+#include "test.h"
+
+/* used as the service core ID */
+static uint32_t score_id;
+/* used as timestamp to detect if a service core is running */
+static uint64_t service_tick;
+
+#define SERVICE_DELAY 1
+
+static int
+testsuite_setup(void)
+{
+	/* assuming lcore 1 is available for service-core testing */
+	score_id = 1;
+	return TEST_SUCCESS;
+}
+
+static void
+testsuite_teardown(void)
+{
+	/* release service cores? */
+}
+
+static int32_t dummy_cb(void *args)
+{
+	RTE_SET_USED(args);
+	service_tick++;
+	rte_delay_ms(SERVICE_DELAY);
+	return 0;
+}
+
+/* unregister all services */
+static int
+dummy_unregister(void)
+{
+	uint32_t i;
+	struct rte_service_spec *dead = (struct rte_service_spec *)0xdead;
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_unregister(0),
+			"Unregistered NULL pointer");
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_unregister(dead),
+			"Unregistered invalid pointer");
+
+	uint32_t c = rte_service_get_count();
+	for (i = 0; i < c; i++) {
+		struct rte_service_spec *s = rte_service_get_by_id(i);
+		TEST_ASSERT_EQUAL(0, rte_service_unregister(s),
+				"Error unregistering a valid service");
+	}
+
+	rte_service_core_reset_all();
+
+	return TEST_SUCCESS;
+}
+
+/* register a single dummy service */
+static int
+dummy_register(void)
+{
+	/* make sure there are no remains from previous tests */
+	dummy_unregister();
+
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid callback");
+	service.callback = dummy_cb;
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid name");
+	snprintf(service.name, sizeof(service.name), "dummy_service");
+
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Failed to register valid service");
+
+	return TEST_SUCCESS;
+}
+
+/* start and stop a service */
+static int
+service_start_stop(void)
+{
+	struct rte_service_spec *service = rte_service_get_by_id(0);
+
+	TEST_ASSERT_EQUAL(0, rte_service_is_running(service),
+			"Error: Service is running - should be stopped");
+
+	TEST_ASSERT_EQUAL(0, rte_service_stop(service),
+			"Error: Service stopped returned non-zero");
+
+	TEST_ASSERT_EQUAL(0, rte_service_is_running(service),
+			"Error: Service is running - should be stopped");
+
+	TEST_ASSERT_EQUAL(0, rte_service_start(service),
+			"Error: Service start returned non-zero");
+
+	TEST_ASSERT_EQUAL(1, rte_service_is_running(service),
+			"Error: Service is not running");
+
+	return dummy_unregister();
+}
+
+/* enable and disable a core for a service */
+static int
+service_core_enable_disable(void)
+{
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+
+	/* expected failure cases */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_enable_on_core(s, 100000),
+			"Enable on invalid core did not fail");
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_disable_on_core(s, 100000),
+			"Dispable on invalid core did not fail");
+
+	/* add service core to allow enabling */
+	TEST_ASSERT_EQUAL(0, rte_service_core_add(score_id),
+			"Add service core failed when not in use before");
+
+	/* valid enable */
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_core(s, score_id),
+			"Enabling valid service and core failed");
+	TEST_ASSERT_EQUAL(1, rte_service_get_enabled_on_core(s, score_id),
+			"Enabled core returned not-enabled");
+
+	/* valid disable */
+	TEST_ASSERT_EQUAL(0, rte_service_disable_on_core(s, score_id),
+			"Disabling valid service and core failed");
+	TEST_ASSERT_EQUAL(0, rte_service_get_enabled_on_core(s, score_id),
+			"Disabled core returned enabled");
+
+	return dummy_unregister();
+}
+
+static int
+service_core_running_check(void)
+{
+	uint64_t tick = service_tick;
+	rte_delay_ms(SERVICE_DELAY * 10);
+	/* if (tick != service_tick) we know the core as polled the service */
+	return tick != service_tick;
+}
+
+static int
+service_core_add_del(void)
+{
+	/* check initial count */
+	TEST_ASSERT_EQUAL(0, rte_service_core_count(),
+			"Service core count has value before adding a core");
+
+	/* check service core add */
+	TEST_ASSERT_EQUAL(0, rte_service_core_add(score_id),
+			"Add service core failed when not in use before");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_core_add(score_id),
+			"Add service core failed to refuse in-use lcore");
+
+	/* check count */
+	TEST_ASSERT_EQUAL(1, rte_service_core_count(),
+			"Service core count not equal to one");
+
+	/* retrieve core list, checking lcore ids */
+	const uint32_t size = 4;
+	uint32_t service_core_ids[size];
+	int32_t n = rte_service_core_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(1, n, "Service core list return should equal 1");
+	TEST_ASSERT_EQUAL(score_id, service_core_ids[0],
+				"Service core list lcore must equal score_id");
+
+	/* recheck count, add more cores, and check count */
+	TEST_ASSERT_EQUAL(1, rte_service_core_count(),
+			"Service core count not equal to one");
+	TEST_ASSERT_EQUAL(0, rte_service_core_add(2),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_core_add(3),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(3, rte_service_core_count(),
+			"Service core count not equal to three");
+
+	/* check longer service core list */
+	n = rte_service_core_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(3, n, "Service core list return should equal 3");
+	TEST_ASSERT_EQUAL(score_id, service_core_ids[0],
+				"Service core list[0] lcore must equal 1");
+	TEST_ASSERT_EQUAL(2, service_core_ids[1],
+				"Service core list[1] lcore must equal 2");
+	TEST_ASSERT_EQUAL(3, service_core_ids[2],
+				"Service core list[2] lcore must equal 3");
+
+	/* recheck count, remove lcores, check remaining lcore_id is correct */
+	TEST_ASSERT_EQUAL(3, rte_service_core_count(),
+			"Service core count not equal to three");
+	TEST_ASSERT_EQUAL(0, rte_service_core_del(1),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_core_del(2),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(1, rte_service_core_count(),
+			"Service core count not equal to one");
+	n = rte_service_core_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(1, n, "Service core list return should equal one");
+	TEST_ASSERT_EQUAL(3, service_core_ids[0],
+				"Service core list[0] lcore must equal three");
+
+	return dummy_unregister();
+}
+
+/* start and stop a service core - ensuring it goes back to sleep */
+static int
+service_core_start_stop(void)
+{
+	/* start service core and service, create mapping so tick() runs */
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+	TEST_ASSERT_EQUAL(0, rte_service_start(s),
+			"Starting valid service failed");
+	/* TODO: should this work? Core isn't enabled as service core */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_enable_on_core(s, score_id),
+			"Enabling valid service on invalid core must fail");
+
+	/* core start */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_core_start(score_id),
+			"Service core start without add should return EINVAL");
+	TEST_ASSERT_EQUAL(0, rte_service_core_add(score_id),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_core(s, score_id),
+			"Enabling valid service on valid core failed");
+	TEST_ASSERT_EQUAL(0, rte_service_core_start(score_id),
+			"Service core start after add failed");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_core_start(score_id),
+			"Service core expected as running but was stopped");
+
+	/* ensures core really is running the service function */
+	TEST_ASSERT_EQUAL(1, service_core_running_check(),
+			"Service core expected to poll service but it didn't");
+
+	/* core stop */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_core_stop(100000),
+			"Invalid Service core stop should return -EINVAL");
+	TEST_ASSERT_EQUAL(0, rte_service_core_stop(score_id),
+			"Service core stop expected to return 0");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_core_stop(score_id),
+			"Already stopped service core should return -EALREADY");
+
+	/* ensure service is not longer running */
+	TEST_ASSERT_EQUAL(0, service_core_running_check(),
+			"Service core expected to poll service but it didn't");
+
+	return dummy_unregister();
+}
+
+static struct unit_test_suite service_tests  = {
+	.suite_name = "service core test suite",
+	.setup = testsuite_setup,
+	.teardown = testsuite_teardown,
+	.unit_test_cases = {
+		TEST_CASE_ST(dummy_register, NULL, dummy_unregister),
+		TEST_CASE_ST(dummy_register, NULL, service_start_stop),
+		TEST_CASE_ST(dummy_register, NULL, service_core_add_del),
+		TEST_CASE_ST(dummy_register, NULL, service_core_start_stop),
+		TEST_CASE_ST(dummy_register, NULL, service_core_enable_disable),
+		TEST_CASES_END() /**< NULL terminate unit test array */
+	}
+};
+
+static int
+test_service_common(void)
+{
+	return unit_test_suite_runner(&service_tests);
+}
+
+REGISTER_TEST_COMMAND(service_autotest, test_service_common);
-- 
2.7.4

[PATCH 6/6] service cores: enable event/sw with service

[Harry van Haaren]

This commit shows how easy it is to enable a specific
DPDK component with a service callback, in order to get
CPU cycles for it.

The beauty of this method is that the service is unaware
of how much CPU time it is getting - the application can
decide how to split and slice cores and map them to the
registered services.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
---
 drivers/event/sw/sw_evdev.c | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
index a31aaa6..f55cad9 100644
--- a/drivers/event/sw/sw_evdev.c
+++ b/drivers/event/sw/sw_evdev.c
@@ -38,6 +38,9 @@
 #include <rte_ring.h>
 #include <rte_errno.h>
 
+#include <rte_service_private.h>
+#include <rte_cycles.h>
+
 #include "sw_evdev.h"
 #include "iq_ring.h"
 #include "event_ring.h"
@@ -695,6 +698,14 @@ set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque)
 	return 0;
 }
 
+
+static int32_t sw_sched_service_func(void *args)
+{
+	struct rte_eventdev *dev = args;
+	sw_event_schedule(dev);
+	return 0;
+}
+
 static int
 sw_probe(struct rte_vdev_device *vdev)
 {
@@ -806,6 +817,17 @@ sw_probe(struct rte_vdev_device *vdev)
 	sw->credit_update_quanta = credit_quanta;
 	sw->sched_quanta = sched_quanta;
 
+	/* register service with EAL */
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	snprintf(service.name, sizeof(service.name), "%s_service", name);
+	service.socket_id = socket_id;
+	service.callback = sw_sched_service_func;
+	service.callback_userdata = (void *)dev;
+
+	int32_t ret = rte_service_register(&service);
+	printf("serivce register = %d, cb ud %p\n", ret, dev);
+
 	return 0;
 }
 
-- 
2.7.4

Re: [PATCH 1/6] service cores: header and implementation

[Jerin Jacob]

-----Original Message-----
> Date: Fri, 23 Jun 2017 10:06:14 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: thomas@monjalon.net, jerin.jacob@caviumnetworks.com,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH 1/6] service cores: header and implementation
> X-Mailer: git-send-email 2.7.4
> 
> Add header files, update .map files with new service
> functions, and add the service header to the doxygen
> for building.
> 
> This service header API allows DPDK to use services as
> a concept of something that requires CPU cycles. An example
> is a PMD that runs in software to schedule events, where a
> hardware version exists that does not require a CPU.
> 
> The code presented here is based on an initial RFC:
> http://dpdk.org/ml/archives/dev/2017-May/065207.html
> 
> This was then reworked, and RFC v2 with the changes posted:
> http://dpdk.org/ml/archives/dev/2017-June/067194.html
> 
> This is the third iteration of the service core concept,
> now with an implementation.
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

Nice work. Detailed review comments below

> ---
>  doc/api/doxy-api-index.md                          |   1 +
>  lib/librte_eal/bsdapp/eal/Makefile                 |   1 +
>  lib/librte_eal/bsdapp/eal/rte_eal_version.map      |  24 +
>  lib/librte_eal/common/Makefile                     |   1 +
>  lib/librte_eal/common/include/rte_eal.h            |   4 +
>  lib/librte_eal/common/include/rte_lcore.h          |   3 +-
>  lib/librte_eal/common/include/rte_service.h        | 274 ++++++++++
>  .../common/include/rte_service_private.h           | 108 ++++
>  lib/librte_eal/common/rte_service.c                | 568 +++++++++++++++++++++
>  lib/librte_eal/linuxapp/eal/Makefile               |   1 +
>  lib/librte_eal/linuxapp/eal/rte_eal_version.map    |  24 +
>  11 files changed, 1008 insertions(+), 1 deletion(-)
>  create mode 100644 lib/librte_eal/common/include/rte_service.h
>  create mode 100644 lib/librte_eal/common/include/rte_service_private.h
>  create mode 100644 lib/librte_eal/common/rte_service.c
> 
> diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
> index f5f1f19..55d522a 100644
> --- a/doc/api/doxy-api-index.md
> +++ b/doc/api/doxy-api-index.md
> @@ -158,6 +158,7 @@ There are many libraries, so their headers may be grouped by topics:
>    [common]             (@ref rte_common.h),
>    [ABI compat]         (@ref rte_compat.h),
>    [keepalive]          (@ref rte_keepalive.h),
> +  [Service Cores]      (@ref rte_service.h),

1) IMO, To keep the consistency we can rename to "[service cores]"
2) I thought, we decided to expose rte_service_register() and
rte_service_unregister() as well, Considering the case where even application
as register for service functions if required. If it is true then I
think, registration functions can moved of private header file so that
it will visible in doxygen.
3) Should we change core function name as lcore like
rte_service_lcore_add(), rte_service_lcore_del() etc as we are operating
on lcore here.


>    [device metrics]     (@ref rte_metrics.h),
>    [bitrate statistics] (@ref rte_bitrate.h),
>    [latency statistics] (@ref rte_latencystats.h),
> diff --git a/lib/librte_eal/bsdapp/eal/Makefile b/lib/librte_eal/bsdapp/eal/Makefile
> index a0f9950..05517a2 100644
> --- a/lib/librte_eal/bsdapp/eal/Makefile
> +++ b/lib/librte_eal/bsdapp/eal/Makefile
> @@ -87,6 +87,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_malloc.c
>  SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_elem.c
>  SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_heap.c
>  SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_keepalive.c
> +SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_service.c
>  
>  # from arch dir
>  SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_cpuflags.c
> diff --git a/lib/librte_eal/bsdapp/eal/rte_eal_version.map b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
> index 2e48a73..843d4ee 100644
> --- a/lib/librte_eal/bsdapp/eal/rte_eal_version.map
> +++ b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
> @@ -193,3 +193,27 @@ DPDK_17.05 {
>  	vfio_get_group_no;
>  
>  } DPDK_17.02;
> +
> +DPDK_17.08 {
> +	global:
> +
> +	rte_service_core_add;
> +	rte_service_core_count;
> +	rte_service_core_del;
> +	rte_service_core_list;
> +	rte_service_core_reset_all;
> +	rte_service_core_start;
> +	rte_service_core_stop;
> +	rte_service_disable_on_core;
> +	rte_service_enable_on_core;
> +	rte_service_get_by_id;
> +	rte_service_get_count;
> +	rte_service_get_enabled_on_core;
> +	rte_service_is_running;
> +	rte_service_register;
> +	rte_service_reset;
> +	rte_service_start;
> +	rte_service_stop;
> +	rte_service_unregister;
> +
> +} DPDK_17.05;
> diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
> index a5bd108..2a93397 100644
> --- a/lib/librte_eal/common/Makefile
> +++ b/lib/librte_eal/common/Makefile
> @@ -41,6 +41,7 @@ INC += rte_eal_memconfig.h rte_malloc_heap.h
>  INC += rte_hexdump.h rte_devargs.h rte_bus.h rte_dev.h rte_vdev.h
>  INC += rte_pci_dev_feature_defs.h rte_pci_dev_features.h
>  INC += rte_malloc.h rte_keepalive.h rte_time.h
> +INC += rte_service.h rte_service_private.h
>  
>  GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h
>  GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h
> diff --git a/lib/librte_eal/common/include/rte_eal.h b/lib/librte_eal/common/include/rte_eal.h
> index abf020b..1f203f8 100644
> --- a/lib/librte_eal/common/include/rte_eal.h
> +++ b/lib/librte_eal/common/include/rte_eal.h
> @@ -61,6 +61,7 @@ extern "C" {
>  enum rte_lcore_role_t {
>  	ROLE_RTE,
>  	ROLE_OFF,
> +	ROLE_SERVICE,
>  };
>  
>  /**
> @@ -80,6 +81,7 @@ enum rte_proc_type_t {
>  struct rte_config {
>  	uint32_t master_lcore;       /**< Id of the master lcore */
>  	uint32_t lcore_count;        /**< Number of available logical cores. */
> +	uint32_t score_count;        /**< Number of available service cores. */

Should we call it as service core or service lcore?

>  	enum rte_lcore_role_t lcore_role[RTE_MAX_LCORE]; /**< State of cores. */
>  
>  	/** Primary or secondary configuration */
> @@ -185,6 +187,8 @@ int rte_eal_iopl_init(void);
>   *
>   *     EPROTO indicates that the PCI bus is either not present, or is not
>   *            readable by the eal.
> + *
> + *     ENOEXEC indicates that a service core failed to launch successfully.
>   */

> +#define RTE_SERVICE_CAP_MT_SAFE (1 << 0)
> +
> +/** Return the number of services registered.
> + *
> + * The number of services registered can be passed to *rte_service_get_by_id*,
> + * enabling the application to retireve the specificaion of each service.

s/retireve the specificaion/retrieve the specification

> + *
> + * @return The number of services registered.
> + */
> +uint32_t rte_service_get_count(void);
> +
> +/** Return the specificaion of each service.

s/specificaion/specification

> + *
> + * This function provides the specification of a service. This can be used by
> + * the application to understand what the service represents. The service
> + * must not be modified by the application directly, only passed to the various
> + * rte_service_* functions.
> + *
> + * @param id The integer id of the service to retrieve
> + * @retval non-zero A valid pointer to the service_spec
> + * @retval NULL Invalid *id* provided.
> + */
> +struct rte_service_spec *rte_service_get_by_id(uint32_t id);
> +
> +/** Return the name of the service.
> + *
> + * @return A pointer to the name of the service. The returned pointer remains
> + *         in ownership of the service, and the application must not free it.
> + */
> +const char *rte_service_get_name(const struct rte_service_spec *service);
> +
> +/* Check if a service has a specific capability.

Missing the doxygen marker(ie. change to /** Check)

> + *
> + * This function returns if *service* has implements *capability*.
> + * See RTE_SERVICE_CAP_* defines for a list of valid capabilities.
> + * @retval 1 Capability supported by this service instance
> + * @retval 0 Capability not supported by this service instance
> + */
> +int32_t rte_service_probe_capability(const struct rte_service_spec *service,
> +				     uint32_t capability);
> +
> +/* Start a service core.

Missing the doxygen marker(ie. change to /** Start)

> + *
> + * Starting a core makes the core begin polling. Any services assigned to it
> + * will be run as fast as possible.
> + *
> + * @retval 0 Success
> + * @retval -EINVAL Failed to start core. The *lcore_id* passed in is not
> + *          currently assigned to be a service core.
> + */
> +int32_t rte_service_core_start(uint32_t lcore_id);
> +
> +/* Stop a service core.

Missing the doxygen marker(ie. change to /** Stop)

> + *
> + * Stopping a core makes the core become idle, but remains  assigned as a
> + * service core.
> + *
> + * @retval 0 Success
> + * @retval -EINVAL Invalid *lcore_id* provided
> + * @retval -EALREADY Already stopped core
> + * @retval -EBUSY Failed to stop core, as it would cause a service to not
> + *          be run, as this is the only core currently running the service.
> + *          The application must stop the service first, and then stop the
> + *          lcore.
> + */
> +int32_t rte_service_core_stop(uint32_t lcore_id);
> +
> +/** Retreve the number of service cores currently avaialble.
typo: ^^^^^^^^                                      ^^^^^^^^^^
Retrieve the number of service cores currently available.

> + *
> + * This function returns the integer count of service cores available. The
> + * service core count can be used in mapping logic when creating mappings
> + * from service cores to services.
> + *
> + * See *rte_service_core_list* for details on retrieving the lcore_id of each
> + * service core.
> + *
> + * @return The number of service cores currently configured.
> + */
> +int32_t rte_service_core_count(void);
> +
> +/** Retrieve the list of currently enabled service cores.
> + *
> + * This function fills in an application supplied array, with each element
> + * indicating the lcore_id of a service core.
> + *
> + * Adding and removing service cores can be performed using
> + * *rte_service_core_add* and *rte_service_core_del*.
> + * @param array An array of at least N items.

@param [out]  array An array of at least n items

> + * @param The size of *array*.

@param n The size of *array*.

> + * @retval >=0 Number of service cores that have been populated in the array
> + * @retval -ENOMEM The provided array is not large enough to fill in the
> + *          service core list. No items have been populated, call this function
> + *          with a size of at least *rte_service_core_count* items.
> + */
> +int32_t rte_service_core_list(uint32_t array[], uint32_t n);
> +
> +/** Dumps any information available about the service. If service is NULL,
> + * dumps info for all services.
> + */
> +int32_t rte_service_dump(FILE *f, struct rte_service_spec *service);
> +
> +#ifdef __cplusplus
> +}
> +#endif
> +
> +
> +#endif /* _RTE_SERVICE_H_ */
> diff --git a/lib/librte_eal/common/include/rte_service_private.h b/lib/librte_eal/common/include/rte_service_private.h
> new file mode 100644
> index 0000000..d8bb644
> --- /dev/null
> +++ b/lib/librte_eal/common/include/rte_service_private.h
> @@ -0,0 +1,108 @@
> +/* This file specifies the internal service specification.
> + * Include this file if you are writing a component that requires CPU cycles to
> + * operate, and you wish to run the component using service cores
> + */
> +
> +#include <rte_service.h>
> +struct rte_service_spec {
> +        /** The name of the service. This should be used by the application to  
> +         * understand what purpose this service provides.
> +         */
> +        char name[RTE_SERVICE_NAME_MAX];
> +        /** The callback to invoke to run one iteration of the service */      
> +        rte_service_func callback;
> +        /** The userdata pointer provided to the service callback. */           
> +        void *callback_userdata;
> +        /** Flags to indicate the capabilities of this service. See
> + * defines in
> +         * the public header file for values of RTE_SERVICE_CAP_*               
> +        */
> +       uint32_t capabilities;
> +       /** NUMA socket ID that this service is affinitized to */               
> +       int8_t socket_id;

All other places socket_id is of type "int". I think, we can maintenance
the consistency here too. Looks like socket_id == SOCKET_ID_ANY not take
care in implementation if so, take care of it.

> +};

> +
> +int32_t rte_service_register(const struct rte_service_spec *spec);
> +
> +/** Unregister a service.
> + *
> + * The service being removed must be stopped before calling this function.
> + *
> + * @retval 0 The service was successfully unregistered.
> + * @retval -EBUSY The service is currently running, stop the service before
> + *          calling unregister. No action has been taken.
> + */
> +int32_t rte_service_unregister(struct rte_service_spec *service);
> +
> +/** Private function to allow EAL to initialied default mappings.

typo:                                   ^^^^^^^^^^^

> + *
> + * This function iterates all the services, and maps then to the available
> + * cores. Based on the capabilities of the services, they are set to run on the
> + * available cores in a round-robin manner.
> + *
> + * @retval 0 Success
> + */
> +int32_t rte_service_init_default_mapping(void);
> +
> +#endif /* _RTE_SERVICE_PRIVATE_H_ */
> diff --git a/lib/librte_eal/common/rte_service.c b/lib/librte_eal/common/rte_service.c
> new file mode 100644
> index 0000000..8b5e344
> --- /dev/null
> +++ b/lib/librte_eal/common/rte_service.c
> +#define RTE_SERVICE_NUM_MAX 64
> +
> +#define RTE_SERVICE_FLAG_REGISTERED_SHIFT 0

Internal macro, Can be shorten to reduce the length(SERVICE_F_REGISTERED?)

> +
> +#define RTE_SERVICE_RUNSTATE_STOPPED 0
> +#define RTE_SERVICE_RUNSTATE_RUNNING 1

Internal macro, Can be shorten to reduce the length(SERVICE_STATE_RUNNING?)


> +
> +/* internal representation of a service */
> +struct rte_service_spec_impl {
> +	/* public part of the struct */
> +	struct rte_service_spec spec;

Nice approach.

> +
> +	/* atomic lock that when set indicates a service core is currently
> +	 * running this service callback. When not set, a core may take the
> +	 * lock and then run the service callback.
> +	 */
> +	rte_atomic32_t execute_lock;
> +
> +	/* API set/get-able variables */
> +	int32_t runstate;
> +	uint8_t internal_flags;
> +
> +	/* per service statistics */
> +	uint32_t num_mapped_cores;
> +	uint64_t calls;
> +	uint64_t cycles_spent;
> +};

Since it been used in fastpath. better to align to cache line

> +
> +/* the internal values of a service core */
> +struct core_state {
> +	uint64_t service_mask; /* map of services IDs are run on this core */
> +	uint8_t runstate; /* running or stopped */
> +	uint8_t is_service_core; /* set if core is currently a service core */
> +
> +	/* extreme statistics */
> +	uint64_t calls_per_service[RTE_SERVICE_NUM_MAX];
> +};

aligned to cache line?

> +
> +static uint32_t rte_service_count;
> +static struct rte_service_spec_impl rte_services[RTE_SERVICE_NUM_MAX];
> +static struct core_state cores_state[RTE_MAX_LCORE];

Since these variable are used in fastpath, better to allocate form
huge page area. It will avoid lot of global variables in code as well.
Like other module, you can add a private function for service init and it can be
called from eal_init()


> +
> +/* returns 1 if service is registered and has not been unregistered
> + * Returns 0 if service never registered, or has been unregistered
> + */
> +static int

static inline int

> +service_valid(uint32_t id) {
> +	return !!(rte_services[id].internal_flags &
> +		 (1 << RTE_SERVICE_FLAG_REGISTERED_SHIFT));
> +}
> +
> +uint32_t
> +rte_service_get_count(void)
> +{
> +	return rte_service_count;
> +}
> +
> +struct rte_service_spec *
> +rte_service_get_by_id(uint32_t id)
> +{
> +	struct rte_service_spec *service = NULL;
> +	if (id < rte_service_count)
> +		service = (struct rte_service_spec *)&rte_services[id];
> +
> +	return service;
> +}
> +
> +const char *
> +rte_service_get_name(const struct rte_service_spec *service)
> +{
> +	return service->name;
> +}
> +
> +int32_t

bool could be enough here

> +rte_service_probe_capability(const struct rte_service_spec *service,
> +			     uint32_t capability)
> +{
> +	return service->capabilities & capability;
> +}
> +
> +int32_t
> +rte_service_is_running(const struct rte_service_spec *spec)
> +{
> +	if (!spec)
> +		return -EINVAL;
> +
> +	const struct rte_service_spec_impl *impl =
> +		(const struct rte_service_spec_impl *)spec;
> +	return impl->runstate == RTE_SERVICE_RUNSTATE_RUNNING;
> +}
> +
> +int32_t
> +rte_service_register(const struct rte_service_spec *spec)
> +{
> +	uint32_t i;
> +	int32_t free_slot = -1;
> +
> +	if (spec->callback == NULL || strlen(spec->name) == 0)
> +		return -EINVAL;
> +
> +	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
> +		if (!service_valid(i)) {
> +			free_slot = i;
> +			break;
> +		}
> +	}
> +
> +	if (free_slot < 0)

	if ((free_slot < 0) || (i == RTE_SERVICE_NUM_MAX))

> +		return -ENOSPC;
> +
> +	struct rte_service_spec_impl *s = &rte_services[free_slot];
> +	s->spec = *spec;
> +	s->internal_flags |= (1 << RTE_SERVICE_FLAG_REGISTERED_SHIFT);
> +
> +	rte_smp_wmb();
> +	rte_service_count++;

IMO, You can move above rte_smp_wmb() here.

> +
> +	return 0;
> +}
> +
> +int32_t
> +rte_service_unregister(struct rte_service_spec *spec)
> +{
> +	struct rte_service_spec_impl *s = NULL;
> +	struct rte_service_spec_impl *spec_impl =
> +		(struct rte_service_spec_impl *)spec;
> +
> +	uint32_t i;
> +	uint32_t service_id;
> +	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
> +		if (&rte_services[i] == spec_impl) {
> +			s = spec_impl;
> +			service_id = i;
> +			break;
> +		}
> +	}
> +
> +	if (!s)
> +		return -EINVAL;
> +
> +	s->internal_flags &= ~(1 << RTE_SERVICE_FLAG_REGISTERED_SHIFT);
> +
> +	for (i = 0; i < RTE_MAX_LCORE; i++)
> +		cores_state[i].service_mask &= ~(1 << service_id);
> +
> +	memset(&rte_services[service_id], 0,
> +			sizeof(struct rte_service_spec_impl));
> +
> +	rte_smp_wmb();
> +	rte_service_count--;

IMO, You can move above rte_smp_wmb() here.

> +
> +	return 0;
> +}
> +
> +int32_t
> +rte_service_start(struct rte_service_spec *service)
> +{
> +	struct rte_service_spec_impl *s =
> +		(struct rte_service_spec_impl *)service;
> +	s->runstate = RTE_SERVICE_RUNSTATE_RUNNING;

Is this function can called from worker thread? if so add rte_smp_wmb()

> +	return 0;
> +}
> +
> +int32_t
> +rte_service_stop(struct rte_service_spec *service)
> +{
> +	struct rte_service_spec_impl *s =
> +		(struct rte_service_spec_impl *)service;
> +	s->runstate = RTE_SERVICE_RUNSTATE_STOPPED;

Is this function can called from worker thread? if so add rte_smp_wmb()

> +	return 0;
> +}
> +
> +static int32_t
> +rte_service_runner_func(void *arg)
> +{
> +	RTE_SET_USED(arg);
> +	uint32_t i;
> +	const int lcore = rte_lcore_id();
> +	struct core_state *cs = &cores_state[lcore];
> +
> +	while (cores_state[lcore].runstate == RTE_SERVICE_RUNSTATE_RUNNING) {
> +		for (i = 0; i < rte_service_count; i++) {
> +			struct rte_service_spec_impl *s = &rte_services[i];
> +			uint64_t service_mask = cs->service_mask;

No need to read in loop, Move it above while loop and add const.
const uint64_t service_mask = cs->service_mask;


> +
> +			if (s->runstate != RTE_SERVICE_RUNSTATE_RUNNING ||
> +					!(service_mask & (1 << i)))
> +				continue;
> +
> +			uint32_t *lock = (uint32_t *)&s->execute_lock;
> +			if (rte_atomic32_cmpset(lock, 0, 1)) {

rte_atomic32 is costly. How about checking RTE_SERVICE_CAP_MT_SAFE
first.

> +				void *userdata = s->spec.callback_userdata;
> +				uint64_t start = rte_rdtsc();
> +				s->spec.callback(userdata);
> +				uint64_t end = rte_rdtsc();
> +
> +				uint64_t spent = end - start;
> +				s->cycles_spent += spent;
> +				s->calls++;
> +				cs->calls_per_service[i]++;

How about enabling the statistics based on some runtime configuration?

> +
> +				rte_atomic32_clear(&s->execute_lock);
> +			}
> +		}
> +		rte_mb();

Do we need full barrier here. Is rte_smp_rmb() inside the loop is
enough?

> +	}
> +
> +	/* mark core as ready to accept work again */
> +	lcore_config[lcore].state = WAIT;
> +
> +	return 0;
> +}
> +
> +int32_t
> +rte_service_core_count(void)
> +{
> +	int32_t count = 0;
> +	uint32_t i;
> +	for (i = 0; i < RTE_MAX_LCORE; i++)
> +		count += cores_state[i].is_service_core;
> +	return count;
> +}
> +
> +int32_t
> +rte_service_core_list(uint32_t array[], uint32_t n)
> +{
> +	uint32_t count = rte_service_core_count();

	if (!array)
		return -EINVAL;

> +	if (count > n)
> +		return -ENOMEM;
> +
> +	uint32_t i;
> +	uint32_t idx = 0;
> +	for (i = 0; i < RTE_MAX_LCORE; i++) {

Are we good if "count" being the upper limit instead of RTE_MAX_LCORE?

> +		struct core_state *cs = &cores_state[i];
> +		if (cs->is_service_core) {
> +			array[idx] = i;
> +			idx++;
> +		}
> +	}
> +
> +	return count;
> +}
> +
> +int32_t
> +rte_service_init_default_mapping(void)
> +{
> +	/* create a default mapping from cores to services, then start the
> +	 * services to make them transparent to unaware applications.
> +	 */
> +	uint32_t i;
> +	int ret;
> +	uint32_t count = rte_service_get_count();
> +	struct rte_config *cfg = rte_eal_get_configuration();
> +
> +	for (i = 0; i < count; i++) {
> +		struct rte_service_spec *s = rte_service_get_by_id(i);
> +		if (!s)
> +			return -EINVAL;
> +
> +		ret = 0;
> +		int j;
> +		for (j = 0; j < RTE_MAX_LCORE; j++) {
> +			/* TODO: add lcore -> service mapping logic here */
> +			if (cfg->lcore_role[j] == ROLE_SERVICE) {
> +				ret = rte_service_enable_on_core(s, j);
> +				if (ret)
> +					rte_panic("Enabling service core %d on service %s failed\n",
> +							j, s->name);

avoid panic in library


> +			}
> +		}
> +
> +		ret = rte_service_start(s);
> +		if (ret)
> +			rte_panic("failed to start service %s\n", s->name);

avoid panic in library

> +	}
> +
> +	return 0;
> +}
> +
> +static int32_t
> +service_update(struct rte_service_spec *service, uint32_t lcore,
> +		uint32_t *set, uint32_t *enabled)
> +{
> +	uint32_t i;
> +	int32_t sid = -1;
> +
> +	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
> +		if ((struct rte_service_spec *)&rte_services[i] == service &&
> +				service_valid(i)) {
> +			sid = i;
> +			break;
> +		}
> +	}
> +
> +	if (sid == -1 || lcore >= RTE_MAX_LCORE)
> +		return -EINVAL;
> +
> +	if (!cores_state[lcore].is_service_core)
> +		return -EINVAL;
> +
> +	if (set) {
> +		if (*set) {
> +			cores_state[lcore].service_mask |=  (1 << sid);
> +			rte_services[sid].num_mapped_cores++;
> +		} else {
> +			cores_state[lcore].service_mask &= ~(1 << sid);
> +			rte_services[sid].num_mapped_cores--;
> +		}
> +	}
> +
> +	if (enabled)
> +		*enabled = (cores_state[lcore].service_mask & (1 << sid));

If the parent functions can be called from worker thread then add
rte_smp_wmb() here.


> +
> +	return 0;
> +}
> +
> +int32_t rte_service_get_enabled_on_core(struct rte_service_spec *service,
> +					uint32_t lcore)
> +{
> +	uint32_t enabled;
> +	int ret = service_update(service, lcore, 0, &enabled);
> +	if (ret == 0)
> +		return enabled;
> +	return -EINVAL;
> +}
> +
> +int32_t
> +rte_service_enable_on_core(struct rte_service_spec *service, uint32_t lcore)
> +{
> +	uint32_t on = 1;
> +	return service_update(service, lcore, &on, 0);
> +}
> +
> +int32_t
> +rte_service_disable_on_core(struct rte_service_spec *service, uint32_t lcore)
> +{
> +	uint32_t off = 0;
> +	return service_update(service, lcore, &off, 0);
> +}
> +
> +int32_t rte_service_core_reset_all(void)
> +{
> +	/* loop over cores, reset all to mask 0 */
> +	uint32_t i;
> +	for (i = 0; i < RTE_MAX_LCORE; i++) {
> +		cores_state[i].service_mask = 0;
> +		cores_state[i].is_service_core = 0;
> +	}
> +
> +	return 0;
> +}
> +
> +int32_t
> +rte_service_core_add(uint32_t lcore)
> +{
> +	if (lcore >= RTE_MAX_LCORE)
> +		return -EINVAL;
> +	if (cores_state[lcore].is_service_core)
> +		return -EALREADY;
> +
> +	lcore_config[lcore].core_role = ROLE_SERVICE;
> +
> +	/* TODO: take from EAL by setting ROLE_SERVICE? */

I think, we need to fix TODO in v2

> +	cores_state[lcore].is_service_core = 1;
> +	cores_state[lcore].service_mask = 0;
> +
> +	return 0;
> +}
> +
> +int32_t
> +rte_service_core_del(uint32_t lcore)
> +{
> +	if (lcore >= RTE_MAX_LCORE)
> +		return -EINVAL;
> +
> +	struct core_state *cs = &cores_state[lcore];
> +	if (!cs->is_service_core)
> +		return -EINVAL;
> +
> +	if (cs->runstate != RTE_SERVICE_RUNSTATE_STOPPED)
> +		return -EBUSY;
> +
> +	lcore_config[lcore].core_role = ROLE_RTE;
> +	cores_state[lcore].is_service_core = 0;
> +	/* TODO: return to EAL by setting ROLE_RTE? */

I think, we need to fix TODO in v2

> +
> +	return 0;
> +}
> +
> +int32_t
> +rte_service_core_start(uint32_t lcore)
> +{
> +	if (lcore >= RTE_MAX_LCORE)
> +		return -EINVAL;
> +
> +	struct core_state *cs = &cores_state[lcore];
> +	if (!cs->is_service_core)
> +		return -EINVAL;
> +
> +	if (cs->runstate == RTE_SERVICE_RUNSTATE_RUNNING)
> +		return -EALREADY;
> +
> +	/* set core to run state first, and then launch otherwise it will
> +	 * return immidiatly as runstate keeps it in the service poll loop

s/immidiatly/immediately

> +	 */
> +	cores_state[lcore].runstate = RTE_SERVICE_RUNSTATE_RUNNING;
> +
> +	int ret = rte_eal_remote_launch(rte_service_runner_func, 0, lcore);
> +	/* returns -EBUSY if the core is already launched, 0 on success */
> +	return ret;

return rte_eal_remote_launch(rte_service_runner_func, 0, lcore);


> +}
> +
> +int32_t
> +rte_service_core_stop(uint32_t lcore)
> +{
> +	if (lcore >= RTE_MAX_LCORE)
> +		return -EINVAL;
> +
> +	if (cores_state[lcore].runstate == RTE_SERVICE_RUNSTATE_STOPPED)
> +		return -EALREADY;
> +
> +	uint32_t i;
> +	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
> +		int32_t enabled = cores_state[i].service_mask & (1 << i);
> +		int32_t service_running = rte_services[i].runstate !=
> +						RTE_SERVICE_RUNSTATE_STOPPED;
> +		int32_t only_core = rte_services[i].num_mapped_cores == 1;
> +
> +		/* if the core is mapped, and the service is running, and this
> +		 * is the only core that is mapped, the service would cease to
> +		 * run if this core stopped, so fail instead.
> +		 */
> +		if (enabled && service_running && only_core)
> +			return -EBUSY;
> +	}
> +
> +	cores_state[lcore].runstate = RTE_SERVICE_RUNSTATE_STOPPED;
> +
> +	return 0;
> +}
> +
> +static void
> +rte_service_dump_one(FILE *f, struct rte_service_spec_impl *s,
> +		     uint64_t all_cycles, uint32_t reset)
> +{
> +	/* avoid divide by zeros */

s/zeros/zero

> +	if (all_cycles == 0)
> +		all_cycles = 1;
> +
> +	int calls = 1;
> +	if (s->calls != 0)
> +		calls = s->calls;
> +
> +	float cycles_pct = (((float)s->cycles_spent) / all_cycles) * 100.f;
> +	fprintf(f,
> +			"  %s : %0.1f %%\tcalls %"PRIu64"\tcycles %"PRIu64"\tavg: %"PRIu64"\n",
> +			s->spec.name, cycles_pct, s->calls, s->cycles_spent,
> +			s->cycles_spent / calls);
> +
> +	if (reset) {
> +		s->cycles_spent = 0;
> +		s->calls = 0;
> +	}
> +}
> +

Re: [PATCH 2/6] service cores: coremask parsing

[Jerin Jacob]

-----Original Message-----
> Date: Fri, 23 Jun 2017 10:06:15 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: thomas@monjalon.net, jerin.jacob@caviumnetworks.com,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH 2/6] service cores: coremask parsing
> X-Mailer: git-send-email 2.7.4
> 
> Add logic for parsing a coremask from EAL, which allows
> the application to be unaware of the cores being taken from
> its coremask.
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> ---
>  lib/librte_eal/common/eal_common_options.c | 78 ++++++++++++++++++++++++++++++
>  1 file changed, 78 insertions(+)
> 
> diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
> index f470195..3599784 100644
> --- a/lib/librte_eal/common/eal_common_options.c
> +++ b/lib/librte_eal/common/eal_common_options.c
> @@ -50,6 +50,7 @@
>  #include <rte_version.h>
>  #include <rte_devargs.h>
>  #include <rte_memcpy.h>
> +#include <rte_service.h>
>  
>  #include "eal_internal_cfg.h"
>  #include "eal_options.h"
> @@ -61,6 +62,7 @@ const char
>  eal_short_options[] =
>  	"b:" /* pci-blacklist */
>  	"c:" /* coremask */
> +	"s:" /* service coremask */
>  	"d:" /* driver */
>  	"h"  /* help */
>  	"l:" /* corelist */

Good to have a corelist variant for service lcore list. May be for future.

> @@ -267,6 +269,75 @@ static int xdigit2val(unsigned char c)
>  }
>  
>  static int
> +eal_parse_service_coremask(const char *coremask)
> +{
> +	struct rte_config *cfg = rte_eal_get_configuration();
> +	int i, j, idx = 0;
> +	unsigned count = 0;
> +	char c;
> +	int val;
> +
> +	if (coremask == NULL)
> +		return -1;
> +	/* Remove all blank characters ahead and after .
> +	 * Remove 0x/0X if exists.
> +	 */
> +	while (isblank(*coremask))
> +		coremask++;
> +	if (coremask[0] == '0' && ((coremask[1] == 'x')
> +		|| (coremask[1] == 'X')))
> +		coremask += 2;
> +	i = strlen(coremask);
> +	while ((i > 0) && isblank(coremask[i - 1]))
> +		i--;
> +
> +	if (i == 0)
> +		return -1;
> +
> +	printf("\n\nRemoving Service Cores from lcore roles now\n\n");

s/printf/RTE_LOG


> +
> +	/* TODO: only scan active cores in coremask */
> +	for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
> +		c = coremask[i];
> +		if (isxdigit(c) == 0) {
> +			/* invalid characters */
> +			return -1;
> +		}
> +		val = xdigit2val(c);
> +		for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE;
> +				j++, idx++) {
> +			if ((1 << j) & val) {
> +				/* TODO: enable flexible master core */
> +				if (idx == 0)
> +					continue;
> +
> +				if (!lcore_config[idx].detected) {
> +					RTE_LOG(ERR, EAL,
> +						"lcore %u unavailable\n", idx);
> +					return -1;
> +				}
> +				//cfg->lcore_role[idx] = ROLE_SERVICE;

remove commented code.

> +				rte_service_core_add(idx);
> +				count++;
> +			}
> +		}
> +	}
> +
> +	for (; i >= 0; i--)
> +		if (coremask[i] != '0')
> +			return -1;
> +
> +	for (; idx < RTE_MAX_LCORE; idx++)
> +		lcore_config[idx].core_index = -1;
> +
> +	if (count == 0)
> +		return -1;
> +
> +	cfg->score_count = count;
> +	return 0;
> +}
> +
> +static int
>  eal_parse_coremask(const char *coremask)
>  {
>  	struct rte_config *cfg = rte_eal_get_configuration();
> @@ -826,6 +897,13 @@ eal_parse_common_option(int opt, const char *optarg,
>  		}
>  		core_parsed = 1;
>  		break;
> +	/* service coremask */
> +	case 's':
> +		if (eal_parse_service_coremask(optarg) < 0) {
> +			RTE_LOG(ERR, EAL, "invalid service coremask\n");
> +			return -1;
> +		}
> +		break;
>  	/* size of memory */
>  	case 'm':
>  		conf->memory = atoi(optarg);
> --

With above change:

Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

Re: [PATCH 3/6] service cores: EAL init changes

[Jerin Jacob]

-----Original Message-----
> Date: Fri, 23 Jun 2017 10:06:16 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: thomas@monjalon.net, jerin.jacob@caviumnetworks.com,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH 3/6] service cores: EAL init changes
> X-Mailer: git-send-email 2.7.4
> 
> This commit shows the changes required in rte_eal_init()
> to transparently launch the service threads. The threads
> are launched into the service worker functions here because
> after rte_eal_init() the application is not gauranteed to
> call any other DPDK API.
> 
> As the registration of services happens at initialization
> time, the services that require CPU time are already available
> when we reach the end of rte_eal_init().
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> ---
>  lib/librte_eal/linuxapp/eal/eal.c | 15 +++++++++++++++
>  1 file changed, 15 insertions(+)
> 
> diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c

Need to add bsdapp implementation for the same.

> index 7c78f2d..4d6ad0e 100644
> --- a/lib/librte_eal/linuxapp/eal/eal.c
> +++ b/lib/librte_eal/linuxapp/eal/eal.c
> @@ -78,6 +78,7 @@
>  #include <rte_version.h>
>  #include <rte_atomic.h>
>  #include <malloc_heap.h>
> +#include <rte_service_private.h>
>  
>  #include "eal_private.h"
>  #include "eal_thread.h"
> @@ -939,6 +940,20 @@ rte_eal_init(int argc, char **argv)
>  		return -1;
>  	}
>  
> +	/* initialize service core threads and default service-core mapping */
> +	struct rte_config *config = rte_eal_get_configuration();
> +	uint32_t service_cores[RTE_MAX_LCORE];
> +	int count = rte_service_core_list(service_cores, RTE_MAX_LCORE);
> +	for (i = 0; i < count; i++) {
> +		config->lcore_role[service_cores[i]] = ROLE_SERVICE;

Can we move this change inside rte_service_core_start() itself?
otherwise every application needs to the same. Right?

> +		rte_service_core_start(service_cores[i]);
> +	}
> +	ret = rte_service_init_default_mapping();
> +	if (ret) {
> +		rte_errno = ENOEXEC;
> +		return -1;
> +	}
> +
>  	rte_eal_mcfg_complete();
>  
>  	return fctret;

With above changes:
Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

> -- 
> 2.7.4
> 

Re: [PATCH 5/6] service core: add unit tests

[Jerin Jacob]

-----Original Message-----
> Date: Fri, 23 Jun 2017 10:06:18 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: thomas@monjalon.net, jerin.jacob@caviumnetworks.com,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH 5/6] service core: add unit tests
> X-Mailer: git-send-email 2.7.4
> 
> Add a bunch of unit tests, to ensure that the service
> core functions are operating as expected.
> 
> As part of these tests a dummy service is registered which
> allows identifying if a service callback has been invoked
> by using the CPU tick counter. This allows identifying if
> functions to start and stop service lcores are actually having
> effect.
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> ---
> +#define SERVICE_DELAY 1
> +
> +static int
> +testsuite_setup(void)
> +{
> +	/* assuming lcore 1 is available for service-core testing */

We can check the number of available lcores is >= 2.

> +	score_id = 1;

How about slcore_id?

> +	return TEST_SUCCESS;
> +}
> +
> +static void
> +testsuite_teardown(void)
> +{
> +	/* release service cores? */
> +}
> +
> +static int32_t dummy_cb(void *args)
> +{
> +	RTE_SET_USED(args);
> +	service_tick++;
> +	rte_delay_ms(SERVICE_DELAY);
> +	return 0;
> +}
> +
> +/* unregister all services */
> +static int
> +dummy_unregister(void)

How about unregister_all(void) as it is un registering all services.

> +{
> +	uint32_t i;
> +	struct rte_service_spec *dead = (struct rte_service_spec *)0xdead;
> +
> +	TEST_ASSERT_EQUAL(-EINVAL, rte_service_unregister(0),
> +			"Unregistered NULL pointer");
> +	TEST_ASSERT_EQUAL(-EINVAL, rte_service_unregister(dead),
> +			"Unregistered invalid pointer");
> +
> +	uint32_t c = rte_service_get_count();
> +	for (i = 0; i < c; i++) {
> +		struct rte_service_spec *s = rte_service_get_by_id(i);
> +		TEST_ASSERT_EQUAL(0, rte_service_unregister(s),
> +				"Error unregistering a valid service");
> +	}
> +
> +	rte_service_core_reset_all();
> +
> +	return TEST_SUCCESS;
> +}
> +
> +/* enable and disable a core for a service */
> +static int
> +service_core_enable_disable(void)
> +{
> +	struct rte_service_spec *s = rte_service_get_by_id(0);
> +
> +	/* expected failure cases */
> +	TEST_ASSERT_EQUAL(-EINVAL, rte_service_enable_on_core(s, 100000),
> +			"Enable on invalid core did not fail");
> +	TEST_ASSERT_EQUAL(-EINVAL, rte_service_disable_on_core(s, 100000),
> +			"Dispable on invalid core did not fail");

s/Dispable/Disable

> +
> +	/* add service core to allow enabling */
> +	TEST_ASSERT_EQUAL(0, rte_service_core_add(score_id),
> +			"Add service core failed when not in use before");
> +
> +	/* valid enable */
> +	TEST_ASSERT_EQUAL(0, rte_service_enable_on_core(s, score_id),
> +			"Enabling valid service and core failed");
> +	TEST_ASSERT_EQUAL(1, rte_service_get_enabled_on_core(s, score_id),
> +			"Enabled core returned not-enabled");
> +
> +	/* valid disable */
> +	TEST_ASSERT_EQUAL(0, rte_service_disable_on_core(s, score_id),
> +			"Disabling valid service and core failed");
> +	TEST_ASSERT_EQUAL(0, rte_service_get_enabled_on_core(s, score_id),
> +			"Disabled core returned enabled");
> +
> +	return dummy_unregister();
> +}
> +
> +static struct unit_test_suite service_tests  = {
> +	.suite_name = "service core test suite",
> +	.setup = testsuite_setup,
> +	.teardown = testsuite_teardown,
> +	.unit_test_cases = {
> +		TEST_CASE_ST(dummy_register, NULL, dummy_unregister),
> +		TEST_CASE_ST(dummy_register, NULL, service_start_stop),
> +		TEST_CASE_ST(dummy_register, NULL, service_core_add_del),
> +		TEST_CASE_ST(dummy_register, NULL, service_core_start_stop),
> +		TEST_CASE_ST(dummy_register, NULL, service_core_enable_disable),

IMO, Following functions/functionality coverage is missing. How about add that
in unit test case.
1) rte_service_get_name()
2) Verify RTE_SERVICE_CAP_MT_SAFE with available lcores
3) rte_service_probe_capability()
4) rte_service_dump()
5) Moving service lcore to/from rte lcore back and forth.

Re: [PATCH 6/6] service cores: enable event/sw with service

[Jerin Jacob]

-----Original Message-----
> Date: Fri, 23 Jun 2017 10:06:19 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: thomas@monjalon.net, jerin.jacob@caviumnetworks.com,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH 6/6] service cores: enable event/sw with service
> X-Mailer: git-send-email 2.7.4
> 
> This commit shows how easy it is to enable a specific
> DPDK component with a service callback, in order to get
> CPU cycles for it.
> 
> The beauty of this method is that the service is unaware
> of how much CPU time it is getting - the application can
> decide how to split and slice cores and map them to the
> registered services.
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> ---
>  drivers/event/sw/sw_evdev.c | 22 ++++++++++++++++++++++
>  1 file changed, 22 insertions(+)
> 
> diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
> index a31aaa6..f55cad9 100644
> --- a/drivers/event/sw/sw_evdev.c
> +++ b/drivers/event/sw/sw_evdev.c
> @@ -38,6 +38,9 @@
>  #include <rte_ring.h>
>  #include <rte_errno.h>
>  
> +#include <rte_service_private.h>
> +#include <rte_cycles.h>

If rte_service_private.h needs the rte_cycles.h then I think it can be
included in rte_service_private.h.

> +
>  #include "sw_evdev.h"
>  #include "iq_ring.h"
>  #include "event_ring.h"
> @@ -695,6 +698,14 @@ set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque)
>  	return 0;
>  }
>  
> +
> +static int32_t sw_sched_service_func(void *args)
> +{
> +	struct rte_eventdev *dev = args;
> +	sw_event_schedule(dev);
> +	return 0;
> +}
> +
>  static int
>  sw_probe(struct rte_vdev_device *vdev)
>  {
> @@ -806,6 +817,17 @@ sw_probe(struct rte_vdev_device *vdev)
>  	sw->credit_update_quanta = credit_quanta;
>  	sw->sched_quanta = sched_quanta;
>  
> +	/* register service with EAL */
> +	struct rte_service_spec service;
> +	memset(&service, 0, sizeof(struct rte_service_spec));
> +	snprintf(service.name, sizeof(service.name), "%s_service", name);

Should we add socket_id as well in the service name?


> +	service.socket_id = socket_id;
> +	service.callback = sw_sched_service_func;
> +	service.callback_userdata = (void *)dev;
> +
> +	int32_t ret = rte_service_register(&service);
> +	printf("serivce register = %d, cb ud %p\n", ret, dev);

sw_pmd_log?

> +
>  	return 0;

Should we also check rte_service_is_running() in sw pmd start function
to verify application did everything to setup the service function on
service lcore? or means for feedback?


>  }
>  
> -- 
> 2.7.4
> 

Re: [PATCH 1/6] service cores: header and implementation

[Van Haaren, Harry]

> From: Jerin Jacob [mailto:jerin.jacob@caviumnetworks.com]
<snip>
> > This is the third iteration of the service core concept,
> > now with an implementation.
> >
> > Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> 
> Nice work. Detailed review comments below

Thanks for the (prompt!) feedback. Mostly agree, comments inline, <snip> lots of noisy code out :)

I have a follow up question on service-core usage, and how we can work e.g Eventdev PMDs into Service cores. I'll kick off a new thread on the mailing list to discuss it.

Patchset v2 on the way soon.


> >    [keepalive]          (@ref rte_keepalive.h),
> > +  [Service Cores]      (@ref rte_service.h),
> 
> 1) IMO, To keep the consistency we can rename to "[service cores]"

Done.


> 2) I thought, we decided to expose rte_service_register() and
> rte_service_unregister() as well, Considering the case where even application
> as register for service functions if required. If it is true then I
> think, registration functions can moved of private header file so that
> it will visible in doxygen.

To avoid bleeding implementation out of the API, this was not done. Services register API is not currently publicly visible - this keeps the API abstraction very powerful. If we decide to make the register-struct public, we lost (almost) all of the API encapsulation, as the struct itself has to be public 

Applications can #include <rte_service_private.h> if they insist - which would provide the functionality as desired, but then the application is aware that it is using DPDK private data structures.

I suggest we leave the service register API as private for this release. We can always move it to public if required - once we are more comfortable with the API and it is more widely implemented. This will help keep API/ABI stability - we don't have the "luxury" of EXPERIMENTAL tag in EAL :D


> 3) Should we change core function name as lcore like
> rte_service_lcore_add(), rte_service_lcore_del() etc as we are operating
> on lcore here.

Yep, done. Added "l" to all core related functions for consistency, so lcore is now used everywhere.


> >  struct rte_config {
> >  	uint32_t master_lcore;       /**< Id of the master lcore */
> >  	uint32_t lcore_count;        /**< Number of available logical cores. */
> > +	uint32_t score_count;        /**< Number of available service cores. */
> 
> Should we call it as service core or service lcore?

Done


> > +/** Return the number of services registered.
> > + *
> > + * The number of services registered can be passed to *rte_service_get_by_id*,
> > + * enabling the application to retireve the specificaion of each service.
> 
> s/retireve the specificaion/retrieve the specification
> 
> > + *
> > + * @return The number of services registered.
> > + */
> > +uint32_t rte_service_get_count(void);
> > +
> > +/** Return the specificaion of each service.
> 
> s/specificaion/specification

Fixed

> > +/* Check if a service has a specific capability.
> Missing the doxygen marker(ie. change to /** Check)

Fixed

> > +/* Start a service core.
> Missing the doxygen marker(ie. change to /** Start)

Fixed


> > +/** Retreve the number of service cores currently avaialble.
> typo: ^^^^^^^^                                      ^^^^^^^^^^
> Retrieve the number of service cores currently available.

Oh my do I have talent for mis-spelling :D Fixed


> > + * @param array An array of at least N items.
> 
> @param [out]  array An array of at least n items
> 
> > + * @param The size of *array*.
> 
> @param n The size of *array*.

Done!


> > +       /** NUMA socket ID that this service is affinitized to */
> > +       int8_t socket_id;
> 
> All other places socket_id is of type "int".

Done


> > +/** Private function to allow EAL to initialied default mappings.
> 
> typo:                                   ^^^^^^^^^^^

Fixed

> > +#define RTE_SERVICE_FLAG_REGISTERED_SHIFT 0
> 
> Internal macro, Can be shorten to reduce the length(SERVICE_F_REGISTERED?)
> 
> > +
> > +#define RTE_SERVICE_RUNSTATE_STOPPED 0
> > +#define RTE_SERVICE_RUNSTATE_RUNNING 1
> 
> Internal macro, Can be shorten to reduce the length(SERVICE_STATE_RUNNING?)

These are used for services and for lcore state, so just used RUNSTATE_RUNNING and RUNSTATE_STOPPED.


> > +struct rte_service_spec_impl {
> > +	/* public part of the struct */
> > +	struct rte_service_spec spec;
> 
> Nice approach.
<snip>
> Since it been used in fastpath. better to align to cache line

Done :)


> > +struct core_state {
<snip>
> aligned to cache line?

Done


> > +static uint32_t rte_service_count;
> > +static struct rte_service_spec_impl rte_services[RTE_SERVICE_NUM_MAX];
> > +static struct core_state cores_state[RTE_MAX_LCORE];
> 
> Since these variable are used in fastpath, better to allocate form
> huge page area. It will avoid lot of global variables in code as well.
> Like other module, you can add a private function for service init and it can be
> called from eal_init()

Yep good point, done.


> > +static int
> 
> static inline int
> > +service_valid(uint32_t id) {
Done


> > +int32_t
> 
> bool could be enough here
> 
> > +rte_service_probe_capability(const struct rte_service_spec *service,
> > +			     uint32_t capability)


Currently the entire API is <stdint.h> only, leaving as is.


> > +int32_t
> > +rte_service_register(const struct rte_service_spec *spec)
> > +{
> > +	uint32_t i;
> > +	int32_t free_slot = -1;
> > +
> > +	if (spec->callback == NULL || strlen(spec->name) == 0)
> > +		return -EINVAL;
> > +
> > +	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
> > +		if (!service_valid(i)) {
> > +			free_slot = i;
> > +			break;
> > +		}
> > +	}
> > +
> > +	if (free_slot < 0)
> 
> 	if ((free_slot < 0) || (i == RTE_SERVICE_NUM_MAX))

Ah - a bug! Nice catch, fixed.


> > +	s->internal_flags |= (1 << RTE_SERVICE_FLAG_REGISTERED_SHIFT);
> > +
> > +	rte_smp_wmb();
> > +	rte_service_count++;
> 
> IMO, You can move above rte_smp_wmb() here.


Perhaps I'm not understanding correctly, but don't we need the writes to the service spec to be completed before allowing other cores to see the extra service count? In short, I think the wmb() is in the right place?


> > +	memset(&rte_services[service_id], 0,
> > +			sizeof(struct rte_service_spec_impl));
> > +
> > +	rte_smp_wmb();
> > +	rte_service_count--;
> 
> IMO, You can move above rte_smp_wmb() here.

I think this part needs refactoring actually;

count--;
wmb();
memset();

Stop cores from seeing service, wmb() to ensure writes complete, then clear internal config?


> > +int32_t
> > +rte_service_start(struct rte_service_spec *service)
> > +{
> > +	struct rte_service_spec_impl *s =
> > +		(struct rte_service_spec_impl *)service;
> > +	s->runstate = RTE_SERVICE_RUNSTATE_RUNNING;
> 
> Is this function can called from worker thread? if so add rte_smp_wmb()

Done

> > +	return 0;
> > +}
> > +
> > +int32_t
> > +rte_service_stop(struct rte_service_spec *service)
> > +{
> > +	struct rte_service_spec_impl *s =
> > +		(struct rte_service_spec_impl *)service;
> > +	s->runstate = RTE_SERVICE_RUNSTATE_STOPPED;
> 
> Is this function can called from worker thread? if so add rte_smp_wmb()

Done


> > +static int32_t
> > +rte_service_runner_func(void *arg)
> > +{
> > +	RTE_SET_USED(arg);
> > +	uint32_t i;
> > +	const int lcore = rte_lcore_id();
> > +	struct core_state *cs = &cores_state[lcore];
> > +
> > +	while (cores_state[lcore].runstate == RTE_SERVICE_RUNSTATE_RUNNING) {
> > +		for (i = 0; i < rte_service_count; i++) {
> > +			struct rte_service_spec_impl *s = &rte_services[i];
> > +			uint64_t service_mask = cs->service_mask;
> 
> No need to read in loop, Move it above while loop and add const.
> const uint64_t service_mask = cs->service_mask;

Yep done, I wonder would a compiler be smart enough.. :)


> > +			uint32_t *lock = (uint32_t *)&s->execute_lock;
> > +			if (rte_atomic32_cmpset(lock, 0, 1)) {
> 
> rte_atomic32 is costly. How about checking RTE_SERVICE_CAP_MT_SAFE
> first.

Yep this was on my scope for optimizing down the line.

 
> > +				void *userdata = s->spec.callback_userdata;
> > +				uint64_t start = rte_rdtsc();
> > +				s->spec.callback(userdata);
> > +				uint64_t end = rte_rdtsc();
> > +
> > +				uint64_t spent = end - start;
> > +				s->cycles_spent += spent;
> > +				s->calls++;
> > +				cs->calls_per_service[i]++;
> 
> How about enabling the statistics based on some runtime configuration?

Good idea - added an API to enable/disable statistics collection.


> > +				rte_atomic32_clear(&s->execute_lock);
> > +			}
> > +		}
> > +		rte_mb();
> 
> Do we need full barrier here. Is rte_smp_rmb() inside the loop is
> enough?

Actually I'm not quite sure why there's a barrier at all.. removed.


> > +	uint32_t i;
> > +	uint32_t idx = 0;
> > +	for (i = 0; i < RTE_MAX_LCORE; i++) {
> 
> Are we good if "count" being the upper limit instead of RTE_MAX_LCORE?

Nope, the cores could be anywhere from 0 to RTE_MAX_LCORE - we gotta scan them all.

> > +		struct core_state *cs = &cores_state[i];
> > +		if (cs->is_service_core) {
> > +			array[idx] = i;
> > +			idx++;
> > +		}
> > +	}
> > +
<snip>
> > +				ret = rte_service_enable_on_core(s, j);
> > +				if (ret)
> > +					rte_panic("Enabling service core %d on service %s failed\n",
> > +							j, s->name);
> 
> avoid panic in library

Done

> > +		ret = rte_service_start(s);
> > +		if (ret)
> > +			rte_panic("failed to start service %s\n", s->name);
> 
> avoid panic in library

Done


> > +static int32_t
> > +service_update(struct rte_service_spec *service, uint32_t lcore,
> > +		uint32_t *set, uint32_t *enabled)
> > +{
<snip>
> 
> If the parent functions can be called from worker thread then add
> rte_smp_wmb() here.

Yes they could, done.


> > +	lcore_config[lcore].core_role = ROLE_SERVICE;
> > +
> > +	/* TODO: take from EAL by setting ROLE_SERVICE? */
> 
> I think, we need to fix TODO in v2

Good point :) done


> > +	lcore_config[lcore].core_role = ROLE_RTE;
> > +	cores_state[lcore].is_service_core = 0;
> > +	/* TODO: return to EAL by setting ROLE_RTE? */
> 
> I think, we need to fix TODO in v2

Done

> > +	/* set core to run state first, and then launch otherwise it will
> > +	 * return immidiatly as runstate keeps it in the service poll loop
> 
> s/immidiatly/immediately

Fixed


> > +	int ret = rte_eal_remote_launch(rte_service_runner_func, 0, lcore);
> > +	/* returns -EBUSY if the core is already launched, 0 on success */
> > +	return ret;
> 
> return rte_eal_remote_launch(rte_service_runner_func, 0, lcore);

I got bitten by this twice - documenting the return values, and making it obvious where they come from is worth the variable IMO. Any compiler will optimize away anyways :)

> > +	/* avoid divide by zeros */
> 
> s/zeros/zero

Fixed!


Thanks for the lengthy review - the code has improved a lot - appreciated.

Re: [PATCH 2/6] service cores: coremask parsing

[Van Haaren, Harry]

> From: Jerin Jacob [mailto:jerin.jacob@caviumnetworks.com]

<snip>

> >  #include "eal_internal_cfg.h"
> >  #include "eal_options.h"
> > @@ -61,6 +62,7 @@ const char
> >  eal_short_options[] =
> >  	"b:" /* pci-blacklist */
> >  	"c:" /* coremask */
> > +	"s:" /* service coremask */
> >  	"d:" /* driver */
> >  	"h"  /* help */
> >  	"l:" /* corelist */
> 
> Good to have a corelist variant for service lcore list. May be for future.

Yep agreed.


> > @@ -267,6 +269,75 @@ static int xdigit2val(unsigned char c)
> >  }
> >
> >  static int
> > +eal_parse_service_coremask(const char *coremask)
> > +{
> > +	struct rte_config *cfg = rte_eal_get_configuration();
> > +	int i, j, idx = 0;
> > +	unsigned count = 0;
> > +	char c;
> > +	int val;
> > +
> > +	if (coremask == NULL)
> > +		return -1;
> > +	/* Remove all blank characters ahead and after .
> > +	 * Remove 0x/0X if exists.
> > +	 */
> > +	while (isblank(*coremask))
> > +		coremask++;
> > +	if (coremask[0] == '0' && ((coremask[1] == 'x')
> > +		|| (coremask[1] == 'X')))
> > +		coremask += 2;
> > +	i = strlen(coremask);
> > +	while ((i > 0) && isblank(coremask[i - 1]))
> > +		i--;
> > +
> > +	if (i == 0)
> > +		return -1;
> > +
> > +	printf("\n\nRemoving Service Cores from lcore roles now\n\n");
> 
> s/printf/RTE_LOG

Removed.


> > +	/* TODO: only scan active cores in coremask */
> > +	for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
> > +		c = coremask[i];
> > +		if (isxdigit(c) == 0) {
> > +			/* invalid characters */
> > +			return -1;
> > +		}
> > +		val = xdigit2val(c);
> > +		for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE;
> > +				j++, idx++) {
> > +			if ((1 << j) & val) {
> > +				/* TODO: enable flexible master core */
> > +				if (idx == 0)
> > +					continue;
> > +
> > +				if (!lcore_config[idx].detected) {
> > +					RTE_LOG(ERR, EAL,
> > +						"lcore %u unavailable\n", idx);
> > +					return -1;
> > +				}
> > +				//cfg->lcore_role[idx] = ROLE_SERVICE;
> 
> remove commented code.

Doh - thanks.

<snip>

Re: [PATCH 3/6] service cores: EAL init changes

[Van Haaren, Harry]

> -----Original Message-----
> From: Jerin Jacob [mailto:jerin.jacob@caviumnetworks.com]

<snip>

> > This commit shows the changes required in rte_eal_init()
> > to transparently launch the service threads. The threads
> > are launched into the service worker functions here because
> > after rte_eal_init() the application is not gauranteed to
> > call any other DPDK API.
> >
> > As the registration of services happens at initialization
> > time, the services that require CPU time are already available
> > when we reach the end of rte_eal_init().
> >
> > Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> > ---
> >  lib/librte_eal/linuxapp/eal/eal.c | 15 +++++++++++++++
> >  1 file changed, 15 insertions(+)
> >
> > diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
> 
> Need to add bsdapp implementation for the same.

Done.

> > +	/* initialize service core threads and default service-core mapping */
> > +	struct rte_config *config = rte_eal_get_configuration();
> > +	uint32_t service_cores[RTE_MAX_LCORE];
> > +	int count = rte_service_core_list(service_cores, RTE_MAX_LCORE);
> > +	for (i = 0; i < count; i++) {
> > +		config->lcore_role[service_cores[i]] = ROLE_SERVICE;
> 
> Can we move this change inside rte_service_core_start() itself?
> otherwise every application needs to the same. Right?

Done, nice, this removes the requirement to access rte_config, which I didn't like having there.

Re: [PATCH 5/6] service core: add unit tests

[Van Haaren, Harry]

> From: Jerin Jacob [mailto:jerin.jacob@caviumnetworks.com]
> Sent: Monday, June 26, 2017 2:06 PM
> To: Van Haaren, Harry <harry.van.haaren@intel.com>
> Cc: dev@dpdk.org; thomas@monjalon.net; Wiles, Keith <keith.wiles@intel.com>; Richardson,
> Bruce <bruce.richardson@intel.com>
> Subject: Re: [PATCH 5/6] service core: add unit tests
> 
> -----Original Message-----
> > Date: Fri, 23 Jun 2017 10:06:18 +0100
> > From: Harry van Haaren <harry.van.haaren@intel.com>
<snip>
> > Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> > ---
> > +#define SERVICE_DELAY 1
> > +
> > +static int
> > +testsuite_setup(void)
> > +{
> > +	/* assuming lcore 1 is available for service-core testing */
> 
> We can check the number of available lcores is >= 2.

The service cores are now added at runtime (to allow checking if add/remove works as expected), so this is obsolete.


> > +	score_id = 1;
> 
> How about slcore_id?

Done.

> > +/* unregister all services */
> > +static int
> > +dummy_unregister(void)
> 
> How about unregister_all(void) as it is un registering all services.

Done.


> > +	/* expected failure cases */
> > +	TEST_ASSERT_EQUAL(-EINVAL, rte_service_enable_on_core(s, 100000),
> > +			"Enable on invalid core did not fail");
> > +	TEST_ASSERT_EQUAL(-EINVAL, rte_service_disable_on_core(s, 100000),
> > +			"Dispable on invalid core did not fail");
> 
> s/Dispable/Disable

Fixed


> IMO, Following functions/functionality coverage is missing. How about add that
> in unit test case.
> 1) rte_service_get_name()

Done

> 2) Verify RTE_SERVICE_CAP_MT_SAFE with available lcores

Done, using an atomic in a dummy multi-thread safe service callback to detect that two threads are allowed into the service callback concurrently.

> 3) rte_service_probe_capability()

Done

> 4) rte_service_dump()

Done


> 5) Moving service lcore to/from rte lcore back and forth.

Already covered in service_lcore_add_del(). The rte_service_lcore_add() assigns ROLE_SERVICE to the lcore in lcore_config, so EAL will be aware of it now being a service core. Similarly, _del() will return the lcore to ROLE_RTE.

Re: [PATCH 6/6] service cores: enable event/sw with service

[Van Haaren, Harry]

> From: Jerin Jacob [mailto:jerin.jacob@caviumnetworks.com]
> Sent: Monday, June 26, 2017 2:47 PM
> To: Van Haaren, Harry <harry.van.haaren@intel.com>
> Cc: dev@dpdk.org; thomas@monjalon.net; Wiles, Keith <keith.wiles@intel.com>; Richardson,
> Bruce <bruce.richardson@intel.com>
> Subject: Re: [PATCH 6/6] service cores: enable event/sw with service
> 
> -----Original Message-----
> > Date: Fri, 23 Jun 2017 10:06:19 +0100
> > From: Harry van Haaren <harry.van.haaren@intel.com>
<snip>
> > +#include <rte_service_private.h>
> > +#include <rte_cycles.h>
> 
> If rte_service_private.h needs the rte_cycles.h then I think it can be
> included in rte_service_private.h.

Actually it was stale, a leftover from during development - removed.


> > +	/* register service with EAL */
> > +	struct rte_service_spec service;
> > +	memset(&service, 0, sizeof(struct rte_service_spec));
> > +	snprintf(service.name, sizeof(service.name), "%s_service", name);
> 
> Should we add socket_id as well in the service name?

The user can name the PMD instance with socket ID if they so wish. The name of the PMD instance must already be unique - so we won't gain anything by adding socket ID there either... I'm not seeing value?

The socket ID is available to the application by reading   struct service_spec.


> > +	service.socket_id = socket_id;
> > +	service.callback = sw_sched_service_func;
> > +	service.callback_userdata = (void *)dev;
> > +
> > +	int32_t ret = rte_service_register(&service);
> > +	printf("serivce register = %d, cb ud %p\n", ret, dev);
> 
> sw_pmd_log?

Removed - little value in seeing pointer values bar during development :)


> > +
> >  	return 0;
> 
> Should we also check rte_service_is_running() in sw pmd start function
> to verify application did everything to setup the service function on
> service lcore? or means for feedback?

Yes good idea - rte_service_is_running will return true if the service callback will be called by at least one service lcore - exactly what we need to check.

This check is implemented in v2, on the way soon. As noted in the first email, I'll kick off a mail about discussing how to best use service-cores, and how to enable applications benefit most from service cores (including not using them if they don't want to :)

[PATCH v2 0/5] service cores: cover letter

[Harry van Haaren]

This patchset introduces service cores to DPDK. A service core
is an lcore that performs functions to abstract away details of
differences in environment of the application.

An example is using the eventdev API, where either a software or hardware
PMD performs scheduling. In the case of the software PMD an lcore is
required to perform scheduling, which means application logic would have
to be aware of the PMD running under the API. To abstract away the
differences in HW / SW PMDs, service cores can run the SW PMD service
without application logic specifying the exact cores to use. Note that
eventdev is only one API that benefits; timers, interrupts handling,
statistics and monitoring, and a range of other infrastructure that
requires a slice of CPU time may all benefit from service cores.

The application is not obliged to manually use the service cores API,
however if an application wishes to use the service cores API for fine
grained control over how the services are run, this is possible. Deciding
between a performance threading-profile and scaled-down profile can be
achieved by advanced usage of service cores and setting the lcore mappings.

Finally, the last patch introduces how a PMD can register a service to run
a function. This is then available (along with any other registered services)
to be run by the service cores.

In hope that this makes DPDK and its core-static way-of-life
a bit more dynamic when running dynamic workloads! :)

Regards, -Harry


Harry van Haaren (5):
  service cores: header and implementation
  service cores: EAL init changes
  service cores: coremask parsing
  service cores: add unit tests
  service cores: enable event/sw with service

 doc/api/doxy-api-index.md                          |   1 +
 drivers/event/sw/sw_evdev.c                        |  32 +
 drivers/event/sw/sw_evdev.h                        |   3 +
 lib/librte_eal/bsdapp/eal/Makefile                 |   1 +
 lib/librte_eal/bsdapp/eal/eal.c                    |  22 +
 lib/librte_eal/bsdapp/eal/rte_eal_version.map      |  28 +
 lib/librte_eal/common/Makefile                     |   1 +
 lib/librte_eal/common/eal_common_lcore.c           |   1 +
 lib/librte_eal/common/eal_common_options.c         |  77 +++
 lib/librte_eal/common/include/rte_eal.h            |   4 +
 lib/librte_eal/common/include/rte_lcore.h          |   3 +-
 lib/librte_eal/common/include/rte_service.h        | 298 +++++++++
 .../common/include/rte_service_private.h           | 118 ++++
 lib/librte_eal/common/rte_service.c                | 671 +++++++++++++++++++++
 lib/librte_eal/linuxapp/eal/Makefile               |   1 +
 lib/librte_eal/linuxapp/eal/eal.c                  |  23 +
 lib/librte_eal/linuxapp/eal/eal_thread.c           |   9 +-
 lib/librte_eal/linuxapp/eal/rte_eal_version.map    |  29 +
 test/test/Makefile                                 |   2 +
 test/test/test_service_cores.c                     | 496 +++++++++++++++
 20 files changed, 1818 insertions(+), 2 deletions(-)
 create mode 100644 lib/librte_eal/common/include/rte_service.h
 create mode 100644 lib/librte_eal/common/include/rte_service_private.h
 create mode 100644 lib/librte_eal/common/rte_service.c
 create mode 100644 test/test/test_service_cores.c

-- 
2.7.4

[PATCH v2 1/5] service cores: header and implementation

[Harry van Haaren]

Add header files, update .map files with new service
functions, and add the service header to the doxygen
for building.

This service header API allows DPDK to use services as
a concept of something that requires CPU cycles. An example
is a PMD that runs in software to schedule events, where a
hardware version exists that does not require a CPU.

The code presented here is based on an initial RFC:
http://dpdk.org/ml/archives/dev/2017-May/065207.html
This was then reworked, and RFC v2 with the changes posted:
http://dpdk.org/ml/archives/dev/2017-June/067194.html

This is the fourth iteration of the service core concept,
with 2 RFCs and this being v2 of the implementation.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

---

v2:
Thanks Jerin for review - below a list your suggested changes;
- Doxygen rename to "service cores" for consistency
- use lcore instead of core for function names
- Fix about 10 typos / seplling msitakse ;)
- Dix doxygen /** comments for functions
- Doxygen @param[out] improvements
- int8_t for socket_id to ordinary int
- Rename MACROS for readability
- Align structs to cache lines
- Allocate fastpath-used data from hugepages
- Added/fixed memory barriers for multi-core scheduling
- Add const to variables, and hoist above loop
- Optimize cmpset atomic if MT_SAFE or only one core mapped
- Statistics collection only when requested
- Add error check for array pointer
- Remove panic() calls from library
- Fix TODO notes from previous patchset

There are also some other changes;
- Checkpatch issues fixed
- .map file updates
- Add rte_service_get_by_name() function
---
 doc/api/doxy-api-index.md                          |   1 +
 lib/librte_eal/bsdapp/eal/Makefile                 |   1 +
 lib/librte_eal/bsdapp/eal/rte_eal_version.map      |  28 +
 lib/librte_eal/common/Makefile                     |   1 +
 lib/librte_eal/common/eal_common_lcore.c           |   1 +
 lib/librte_eal/common/include/rte_eal.h            |   4 +
 lib/librte_eal/common/include/rte_lcore.h          |   3 +-
 lib/librte_eal/common/include/rte_service.h        | 298 +++++++++
 .../common/include/rte_service_private.h           | 118 ++++
 lib/librte_eal/common/rte_service.c                | 671 +++++++++++++++++++++
 lib/librte_eal/linuxapp/eal/Makefile               |   1 +
 lib/librte_eal/linuxapp/eal/eal_thread.c           |   9 +-
 lib/librte_eal/linuxapp/eal/rte_eal_version.map    |  29 +
 13 files changed, 1163 insertions(+), 2 deletions(-)
 create mode 100644 lib/librte_eal/common/include/rte_service.h
 create mode 100644 lib/librte_eal/common/include/rte_service_private.h
 create mode 100644 lib/librte_eal/common/rte_service.c

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index f5f1f19..1284402 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -158,6 +158,7 @@ There are many libraries, so their headers may be grouped by topics:
   [common]             (@ref rte_common.h),
   [ABI compat]         (@ref rte_compat.h),
   [keepalive]          (@ref rte_keepalive.h),
+  [service cores]      (@ref rte_service.h),
   [device metrics]     (@ref rte_metrics.h),
   [bitrate statistics] (@ref rte_bitrate.h),
   [latency statistics] (@ref rte_latencystats.h),
diff --git a/lib/librte_eal/bsdapp/eal/Makefile b/lib/librte_eal/bsdapp/eal/Makefile
index a0f9950..05517a2 100644
--- a/lib/librte_eal/bsdapp/eal/Makefile
+++ b/lib/librte_eal/bsdapp/eal/Makefile
@@ -87,6 +87,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_malloc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_elem.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_heap.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_keepalive.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_service.c
 
 # from arch dir
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_cpuflags.c
diff --git a/lib/librte_eal/bsdapp/eal/rte_eal_version.map b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
index 2e48a73..5493a13 100644
--- a/lib/librte_eal/bsdapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
@@ -193,3 +193,31 @@ DPDK_17.05 {
 	vfio_get_group_no;
 
 } DPDK_17.02;
+
+DPDK_17.08 {
+	global:
+
+	rte_service_disable_on_lcore;
+	rte_service_dump;
+	rte_service_enable_on_lcore;
+	rte_service_get_by_id;
+	rte_service_get_by_name;
+	rte_service_get_count;
+	rte_service_get_enabled_on_lcore;
+	rte_service_is_running;
+	rte_service_lcore_add;
+	rte_service_lcore_count;
+	rte_service_lcore_del;
+	rte_service_lcore_list;
+	rte_service_lcore_reset_all;
+	rte_service_lcore_start;
+	rte_service_lcore_stop;
+	rte_service_probe_capability;
+	rte_service_register;
+	rte_service_reset;
+	rte_service_set_stats_enable;
+	rte_service_start;
+	rte_service_stop;
+	rte_service_unregister;
+
+} DPDK_17.05;
diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
index a5bd108..2a93397 100644
--- a/lib/librte_eal/common/Makefile
+++ b/lib/librte_eal/common/Makefile
@@ -41,6 +41,7 @@ INC += rte_eal_memconfig.h rte_malloc_heap.h
 INC += rte_hexdump.h rte_devargs.h rte_bus.h rte_dev.h rte_vdev.h
 INC += rte_pci_dev_feature_defs.h rte_pci_dev_features.h
 INC += rte_malloc.h rte_keepalive.h rte_time.h
+INC += rte_service.h rte_service_private.h
 
 GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h
 GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h
diff --git a/lib/librte_eal/common/eal_common_lcore.c b/lib/librte_eal/common/eal_common_lcore.c
index 84fa0cb..0db1555 100644
--- a/lib/librte_eal/common/eal_common_lcore.c
+++ b/lib/librte_eal/common/eal_common_lcore.c
@@ -81,6 +81,7 @@ rte_eal_cpu_init(void)
 
 		/* By default, each detected core is enabled */
 		config->lcore_role[lcore_id] = ROLE_RTE;
+		lcore_config[lcore_id].core_role = ROLE_RTE;
 		lcore_config[lcore_id].core_id = eal_cpu_core_id(lcore_id);
 		lcore_config[lcore_id].socket_id = eal_cpu_socket_id(lcore_id);
 		if (lcore_config[lcore_id].socket_id >= RTE_MAX_NUMA_NODES) {
diff --git a/lib/librte_eal/common/include/rte_eal.h b/lib/librte_eal/common/include/rte_eal.h
index abf020b..4dd0518 100644
--- a/lib/librte_eal/common/include/rte_eal.h
+++ b/lib/librte_eal/common/include/rte_eal.h
@@ -61,6 +61,7 @@ extern "C" {
 enum rte_lcore_role_t {
 	ROLE_RTE,
 	ROLE_OFF,
+	ROLE_SERVICE,
 };
 
 /**
@@ -80,6 +81,7 @@ enum rte_proc_type_t {
 struct rte_config {
 	uint32_t master_lcore;       /**< Id of the master lcore */
 	uint32_t lcore_count;        /**< Number of available logical cores. */
+	uint32_t service_lcore_count;/**< Number of available service cores. */
 	enum rte_lcore_role_t lcore_role[RTE_MAX_LCORE]; /**< State of cores. */
 
 	/** Primary or secondary configuration */
@@ -185,6 +187,8 @@ int rte_eal_iopl_init(void);
  *
  *     EPROTO indicates that the PCI bus is either not present, or is not
  *            readable by the eal.
+ *
+ *     ENOEXEC indicates that a service core failed to launch successfully.
  */
 int rte_eal_init(int argc, char **argv);
 
diff --git a/lib/librte_eal/common/include/rte_lcore.h b/lib/librte_eal/common/include/rte_lcore.h
index fe7b586..50e0d0f 100644
--- a/lib/librte_eal/common/include/rte_lcore.h
+++ b/lib/librte_eal/common/include/rte_lcore.h
@@ -73,6 +73,7 @@ struct lcore_config {
 	unsigned core_id;          /**< core number on socket for this lcore */
 	int core_index;            /**< relative index, starting from 0 */
 	rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
+	uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
 };
 
 /**
@@ -175,7 +176,7 @@ rte_lcore_is_enabled(unsigned lcore_id)
 	struct rte_config *cfg = rte_eal_get_configuration();
 	if (lcore_id >= RTE_MAX_LCORE)
 		return 0;
-	return cfg->lcore_role[lcore_id] != ROLE_OFF;
+	return cfg->lcore_role[lcore_id] == ROLE_RTE;
 }
 
 /**
diff --git a/lib/librte_eal/common/include/rte_service.h b/lib/librte_eal/common/include/rte_service.h
new file mode 100644
index 0000000..3be59ea
--- /dev/null
+++ b/lib/librte_eal/common/include/rte_service.h
@@ -0,0 +1,298 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_SERVICE_H_
+#define _RTE_SERVICE_H_
+
+/**
+ * @file
+ *
+ * Service functions
+ *
+ * The service functionality provided by this header allows a DPDK component
+ * to indicate that it requires a function call in order for it to perform
+ * its processing.
+ *
+ * An example usage of this functionality would be a component that registers
+ * a service to perform a particular packet processing duty: for example the
+ * eventdev software PMD. At startup the application requests all services
+ * that have been registered, and the cores in the service-coremask run the
+ * required services. The EAL removes these number of cores from the available
+ * runtime cores, and dedicates them to performing service-core workloads. The
+ * application has access to the remaining lcores as normal.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include<stdio.h>
+#include <stdint.h>
+#include <sys/queue.h>
+
+#include <rte_lcore.h>
+
+/* forward declaration only. Definition in rte_service_private.h */
+struct rte_service_spec;
+
+#define RTE_SERVICE_NAME_MAX 32
+
+/* Capabilities of a service.
+ *
+ * Use the *rte_service_probe_capability* function to check if a service is
+ * capable of a specific capability.
+ */
+/** When set, the service is capable of having multiple threads run it at the
+ *  same time.
+ */
+#define RTE_SERVICE_CAP_MT_SAFE (1 << 0)
+
+/** Return the number of services registered.
+ *
+ * The number of services registered can be passed to *rte_service_get_by_id*,
+ * enabling the application to retrieve the specification of each service.
+ *
+ * @return The number of services registered.
+ */
+uint32_t rte_service_get_count(void);
+
+/** Return the specification of a service by integer id.
+ *
+ * This function provides the specification of a service. This can be used by
+ * the application to understand what the service represents. The service
+ * must not be modified by the application directly, only passed to the various
+ * rte_service_* functions.
+ *
+ * @param id The integer id of the service to retrieve
+ * @retval non-zero A valid pointer to the service_spec
+ * @retval NULL Invalid *id* provided.
+ */
+struct rte_service_spec *rte_service_get_by_id(uint32_t id);
+
+/** Return the specification of a service by name.
+ *
+ * This function provides the specification of a service using the service name
+ * as lookup key. This can be used by the application to understand what the
+ * service represents. The service must not be modified by the application
+ * directly, only passed to the various rte_service_* functions.
+ *
+ * @param name The name of the service to retrieve
+ * @retval non-zero A valid pointer to the service_spec
+ * @retval NULL Invalid *name* provided.
+ */
+struct rte_service_spec *rte_service_get_by_name(const char *name);
+
+/** Return the name of the service.
+ *
+ * @return A pointer to the name of the service. The returned pointer remains
+ *         in ownership of the service, and the application must not free it.
+ */
+const char *rte_service_get_name(const struct rte_service_spec *service);
+
+/** Check if a service has a specific capability.
+ *
+ * This function returns if *service* has implements *capability*.
+ * See RTE_SERVICE_CAP_* defines for a list of valid capabilities.
+ * @retval 1 Capability supported by this service instance
+ * @retval 0 Capability not supported by this service instance
+ */
+int32_t rte_service_probe_capability(const struct rte_service_spec *service,
+				     uint32_t capability);
+
+/** Enable a core to run a service.
+ *
+ * Each core can be added or removed from running specific services. This
+ * functions adds *lcore* to the set of cores that will run *service*.
+ *
+ * If multiple cores are enabled on a service, an atomic is used to ensure that
+ * only one cores runs the service at a time. The exception to this is when
+ * a service indicates that it is multi-thread safe by setting the capability
+ * called RTE_SERVICE_CAP_MT_SAFE. With the multi-thread safe capability set,
+ * the service function can be run on multiple threads at the same time.
+ *
+ * @retval 0 lcore added successfully
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_enable_on_lcore(struct rte_service_spec *service,
+				   uint32_t lcore);
+
+/** Disable a core to run a service.
+ *
+ * Each core can be added or removed from running specific services. This
+ * functions removes *lcore* to the set of cores that will run *service*.
+ *
+ * @retval 0 Lcore removed successfully
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_disable_on_lcore(struct rte_service_spec *service,
+				   uint32_t lcore);
+
+/** Return if an lcore is enabled for the service.
+ *
+ * This function allows the application to query if *lcore* is currently set to
+ * run *service*.
+ *
+ * @retval 1 Lcore enabled on this lcore
+ * @retval 0 Lcore disabled on this lcore
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_get_enabled_on_lcore(struct rte_service_spec *service,
+					uint32_t lcore);
+
+
+/** Enable *service* to run.
+ *
+ * This function switches on a service during runtime.
+ * @retval 0 The service was successfully started
+ */
+int32_t rte_service_start(struct rte_service_spec *service);
+
+/** Disable *service*.
+ *
+ * Switch off a service, so it is not run until it is *rte_service_start* is
+ * called on it.
+ * @retval 0 Service successfully switched off
+ */
+int32_t rte_service_stop(struct rte_service_spec *service);
+
+/** Returns if *service* is currently running.
+ *
+ * This function retuns true if the service has been started using
+ * *rte_service_start*, AND a service core is mapped to the service. This
+ * function can be used to ensure that the service will be run.
+ *
+ * @retval 1 Service is currently running, and has a service lcore mapped
+ * @retval 0 Service is currently stopped, or no service lcore is mapped
+ * @retval -EINVAL Invalid service pointer provided
+ */
+int32_t rte_service_is_running(const struct rte_service_spec *service);
+
+/** Start a service core.
+ *
+ * Starting a core makes the core begin polling. Any services assigned to it
+ * will be run as fast as possible.
+ *
+ * @retval 0 Success
+ * @retval -EINVAL Failed to start core. The *lcore_id* passed in is not
+ *          currently assigned to be a service core.
+ */
+int32_t rte_service_lcore_start(uint32_t lcore_id);
+
+/** Stop a service core.
+ *
+ * Stopping a core makes the core become idle, but remains  assigned as a
+ * service core.
+ *
+ * @retval 0 Success
+ * @retval -EINVAL Invalid *lcore_id* provided
+ * @retval -EALREADY Already stopped core
+ * @retval -EBUSY Failed to stop core, as it would cause a service to not
+ *          be run, as this is the only core currently running the service.
+ *          The application must stop the service first, and then stop the
+ *          lcore.
+ */
+int32_t rte_service_lcore_stop(uint32_t lcore_id);
+
+/** Adds lcore to the list of service cores.
+ *
+ * This functions can be used at runtime in order to modify the service core
+ * mask.
+ *
+ * @retval 0 Success
+ * @retval -EBUSY lcore is busy, and not available for service core duty
+ * @retval -EALREADY lcore is already added to the service core list
+ * @retval -EINVAL Invalid lcore provided
+ */
+int32_t rte_service_lcore_add(uint32_t lcore);
+
+/** Removes lcore from the list of service cores.
+ *
+ * This can fail if the core is not stopped, see *rte_service_core_stop*.
+ *
+ * @retval 0 Success
+ * @retval -EBUSY Lcore is not stopped, stop service core before removing.
+ * @retval -EINVAL failed to add lcore to service core mask.
+ */
+int32_t rte_service_lcore_del(uint32_t lcore);
+
+/** Retrieve the number of service cores currently available.
+ *
+ * This function returns the integer count of service cores available. The
+ * service core count can be used in mapping logic when creating mappings
+ * from service cores to services.
+ *
+ * See *rte_service_lcore_list* for details on retrieving the lcore_id of each
+ * service core.
+ *
+ * @return The number of service cores currently configured.
+ */
+int32_t rte_service_lcore_count(void);
+
+/** Reset all service core mappings.
+ * @retval 0 Success
+ */
+int32_t rte_service_lcore_reset_all(void);
+
+/** Enable or disable statistics collection.
+ *
+ * This function enables per core, per-service cycle count collection.
+ * @param enabled Zero to turn off statistics collection, non-zero to enable.
+ */
+void rte_service_set_stats_enable(int enabled);
+
+/** Retrieve the list of currently enabled service cores.
+ *
+ * This function fills in an application supplied array, with each element
+ * indicating the lcore_id of a service core.
+ *
+ * Adding and removing service cores can be performed using
+ * *rte_service_lcore_add* and *rte_service_lcore_del*.
+ * @param [out] array An array of at least N items.
+ * @param [out] The size of *array*.
+ * @retval >=0 Number of service cores that have been populated in the array
+ * @retval -ENOMEM The provided array is not large enough to fill in the
+ *          service core list. No items have been populated, call this function
+ *          with a size of at least *rte_service_core_count* items.
+ */
+int32_t rte_service_lcore_list(uint32_t array[], uint32_t n);
+
+/** Dumps any information available about the service. If service is NULL,
+ * dumps info for all services.
+ */
+int32_t rte_service_dump(FILE *f, struct rte_service_spec *service);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* _RTE_SERVICE_H_ */
diff --git a/lib/librte_eal/common/include/rte_service_private.h b/lib/librte_eal/common/include/rte_service_private.h
new file mode 100644
index 0000000..d518b02
--- /dev/null
+++ b/lib/librte_eal/common/include/rte_service_private.h
@@ -0,0 +1,118 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_SERVICE_PRIVATE_H_
+#define _RTE_SERVICE_PRIVATE_H_
+
+/* This file specifies the internal service specification.
+ * Include this file if you are writing a component that requires CPU cycles to
+ * operate, and you wish to run the component using service cores
+ */
+
+#include <rte_service.h>
+
+/**
+ * Signature of callback function to run a service.
+ */
+typedef int32_t (*rte_service_func)(void *args);
+
+/**
+ * The specification of a service.
+ *
+ * This struct contains metadata about the service itself, the callback
+ * function to run one iteration of the service, a userdata pointer, flags etc.
+ */
+struct rte_service_spec {
+	/** The name of the service. This should be used by the application to
+	 * understand what purpose this service provides.
+	 */
+	char name[RTE_SERVICE_NAME_MAX];
+	/** The callback to invoke to run one iteration of the service. */
+	rte_service_func callback;
+	/** The userdata pointer provided to the service callback. */
+	void *callback_userdata;
+	/** Flags to indicate the capabilities of this service. See defines in
+	 * the public header file for values of RTE_SERVICE_CAP_*
+	 */
+	uint32_t capabilities;
+	/** NUMA socket ID that this service is affinitized to */
+	int socket_id;
+};
+
+/** Register a new service.
+ *
+ * A service represents a component that the requires CPU time periodically to
+ * achieve its purpose.
+ *
+ * For example the eventdev SW PMD requires CPU cycles to perform its
+ * scheduling. This can be achieved by registering it as a service, and the
+ * application can then assign CPU resources to it using
+ * *rte_service_set_coremask*.
+ *
+ * @param spec The specification of the service to register
+ * @retval 0 Successfully registered the service.
+ *         -EINVAL Attempted to register an invalid service (eg, no callback
+ *         set)
+ */
+int32_t rte_service_register(const struct rte_service_spec *spec);
+
+/** Unregister a service.
+ *
+ * The service being removed must be stopped before calling this function.
+ *
+ * @retval 0 The service was successfully unregistered.
+ * @retval -EBUSY The service is currently running, stop the service before
+ *          calling unregister. No action has been taken.
+ */
+int32_t rte_service_unregister(struct rte_service_spec *service);
+
+/** Private function to allow EAL to initialized default mappings.
+ *
+ * This function iterates all the services, and maps then to the available
+ * cores. Based on the capabilities of the services, they are set to run on the
+ * available cores in a round-robin manner.
+ *
+ * @retval 0 Success
+ */
+int32_t rte_service_set_default_mapping(void);
+
+/** Initialize the service library.
+ *
+ * In order to use the service library, it must be initialized. EAL initializes
+ * the library at startup.
+ *
+ * @retval 0 Success
+ * @retval -EALREADY Service library is already initialized
+ */
+int32_t rte_service_init(void);
+
+#endif /* _RTE_SERVICE_PRIVATE_H_ */
diff --git a/lib/librte_eal/common/rte_service.c b/lib/librte_eal/common/rte_service.c
new file mode 100644
index 0000000..67338db
--- /dev/null
+++ b/lib/librte_eal/common/rte_service.c
@@ -0,0 +1,671 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_service.h>
+#include "include/rte_service_private.h"
+
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_common.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_atomic.h>
+#include <rte_memory.h>
+#include <rte_malloc.h>
+
+#define RTE_SERVICE_NUM_MAX 64
+
+#define SERVICE_F_REGISTERED 0
+
+/* runstates for services and lcores, denoting if they are active or not */
+#define RUNSTATE_STOPPED 0
+#define RUNSTATE_RUNNING 1
+
+/* internal representation of a service */
+struct rte_service_spec_impl {
+	/* public part of the struct */
+	struct rte_service_spec spec;
+
+	/* atomic lock that when set indicates a service core is currently
+	 * running this service callback. When not set, a core may take the
+	 * lock and then run the service callback.
+	 */
+	rte_atomic32_t execute_lock;
+
+	/* API set/get-able variables */
+	int32_t runstate;
+	uint8_t internal_flags;
+
+	/* per service statistics */
+	uint32_t num_mapped_cores;
+	uint64_t calls;
+	uint64_t cycles_spent;
+} __rte_cache_aligned;
+
+/* the internal values of a service core */
+struct core_state {
+	/* map of services IDs are run on this core */
+	uint64_t service_mask;
+	uint8_t runstate; /* running or stopped */
+	uint8_t is_service_core; /* set if core is currently a service core */
+	uint8_t collect_statistics; /* if set, measure cycle counts */
+
+	/* extreme statistics */
+	uint64_t calls_per_service[RTE_SERVICE_NUM_MAX];
+} __rte_cache_aligned;
+
+static uint32_t rte_service_count;
+static struct rte_service_spec_impl *rte_services;
+static struct core_state *cores_state;
+static uint32_t rte_service_library_initialized;
+
+int32_t rte_service_init(void)
+{
+	if (rte_service_library_initialized) {
+		printf("service library init() called, init flag %d\n",
+			rte_service_library_initialized);
+		return -EALREADY;
+	}
+
+	rte_services = rte_calloc("rte_services", RTE_SERVICE_NUM_MAX,
+			sizeof(struct rte_service_spec_impl),
+			RTE_CACHE_LINE_SIZE);
+	if (!rte_services) {
+		printf("error allocating rte services array\n");
+		return -ENOMEM;
+	}
+
+	cores_state = rte_calloc("rte_service_core_states", RTE_MAX_LCORE,
+			sizeof(struct core_state), RTE_CACHE_LINE_SIZE);
+	if (!cores_state) {
+		printf("error allocating core states array\n");
+		return -ENOMEM;
+	}
+
+	int i;
+	int count = 0;
+	struct rte_config *cfg = rte_eal_get_configuration();
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		if (lcore_config[i].core_role == ROLE_SERVICE) {
+			if ((unsigned)i == cfg->master_lcore)
+				continue;
+			rte_service_lcore_add(i);
+			count++;
+		}
+	}
+
+	rte_service_library_initialized = 1;
+	return 0;
+}
+
+void rte_service_set_stats_enable(int enabled)
+{
+	uint32_t i;
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		cores_state[i].collect_statistics = enabled;
+}
+
+/* returns 1 if service is registered and has not been unregistered
+ * Returns 0 if service never registered, or has been unregistered
+ */
+static inline int
+service_valid(uint32_t id) {
+	return !!(rte_services[id].internal_flags &
+		 (1 << SERVICE_F_REGISTERED));
+}
+
+uint32_t
+rte_service_get_count(void)
+{
+	return rte_service_count;
+}
+
+struct rte_service_spec *
+rte_service_get_by_id(uint32_t id)
+{
+	struct rte_service_spec *service = NULL;
+	if (id < rte_service_count)
+		service = (struct rte_service_spec *)&rte_services[id];
+
+	return service;
+}
+
+struct rte_service_spec *rte_service_get_by_name(const char *name)
+{
+	struct rte_service_spec *service = NULL;
+	int i;
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (service_valid(i) &&
+				strcmp(name, rte_services[i].spec.name) == 0)
+			service = (struct rte_service_spec *)&rte_services[i];
+			break;
+	}
+
+	return service;
+}
+
+const char *
+rte_service_get_name(const struct rte_service_spec *service)
+{
+	return service->name;
+}
+
+int32_t
+rte_service_probe_capability(const struct rte_service_spec *service,
+			     uint32_t capability)
+{
+	return service->capabilities & capability;
+}
+
+int32_t
+rte_service_is_running(const struct rte_service_spec *spec)
+{
+	const struct rte_service_spec_impl *impl =
+		(const struct rte_service_spec_impl *)spec;
+	if (!impl)
+		return -EINVAL;
+
+	return (impl->runstate == RUNSTATE_RUNNING) &&
+		(impl->num_mapped_cores > 0);
+}
+
+int32_t
+rte_service_register(const struct rte_service_spec *spec)
+{
+	uint32_t i;
+	int32_t free_slot = -1;
+
+	if (spec->callback == NULL || strlen(spec->name) == 0)
+		return -EINVAL;
+
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (!service_valid(i)) {
+			free_slot = i;
+			break;
+		}
+	}
+
+	if ((free_slot < 0) || (i == RTE_SERVICE_NUM_MAX))
+		return -ENOSPC;
+
+	struct rte_service_spec_impl *s = &rte_services[free_slot];
+	s->spec = *spec;
+	s->internal_flags |= (1 << SERVICE_F_REGISTERED);
+
+	rte_smp_wmb();
+	rte_service_count++;
+
+	return 0;
+}
+
+int32_t
+rte_service_unregister(struct rte_service_spec *spec)
+{
+	struct rte_service_spec_impl *s = NULL;
+	struct rte_service_spec_impl *spec_impl =
+		(struct rte_service_spec_impl *)spec;
+
+	uint32_t i;
+	uint32_t service_id;
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (&rte_services[i] == spec_impl) {
+			s = spec_impl;
+			service_id = i;
+			break;
+		}
+	}
+
+	if (!s)
+		return -EINVAL;
+
+	rte_service_count--;
+	rte_smp_wmb();
+
+	s->internal_flags &= ~(1 << SERVICE_F_REGISTERED);
+
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		cores_state[i].service_mask &= ~(1 << service_id);
+
+	memset(&rte_services[service_id], 0,
+			sizeof(struct rte_service_spec_impl));
+
+	return 0;
+}
+
+int32_t
+rte_service_start(struct rte_service_spec *service)
+{
+	struct rte_service_spec_impl *s =
+		(struct rte_service_spec_impl *)service;
+	s->runstate = RUNSTATE_RUNNING;
+	rte_smp_wmb();
+	return 0;
+}
+
+int32_t
+rte_service_stop(struct rte_service_spec *service)
+{
+	struct rte_service_spec_impl *s =
+		(struct rte_service_spec_impl *)service;
+	s->runstate = RUNSTATE_STOPPED;
+	rte_smp_wmb();
+	return 0;
+}
+
+static int32_t
+rte_service_runner_func(void *arg)
+{
+	RTE_SET_USED(arg);
+	uint32_t i;
+	const int lcore = rte_lcore_id();
+	struct core_state *cs = &cores_state[lcore];
+
+	while (cores_state[lcore].runstate == RUNSTATE_RUNNING) {
+		const uint64_t service_mask = cs->service_mask;
+		for (i = 0; i < rte_service_count; i++) {
+			struct rte_service_spec_impl *s = &rte_services[i];
+			if (s->runstate != RUNSTATE_RUNNING ||
+					!(service_mask & (1 << i)))
+				continue;
+
+			/* check if this is the only core mapped, else use
+			 * atomic to serialize cores mapped to this service
+			 */
+			uint32_t *lock = (uint32_t *)&s->execute_lock;
+			if ((s->spec.capabilities & RTE_SERVICE_CAP_MT_SAFE) ||
+					(s->num_mapped_cores == 1 ||
+					rte_atomic32_cmpset(lock, 0, 1))) {
+				void *userdata = s->spec.callback_userdata;
+
+				if (cs->collect_statistics) {
+					uint64_t start = rte_rdtsc();
+					s->spec.callback(userdata);
+					uint64_t end = rte_rdtsc();
+					s->cycles_spent += end - start;
+					cs->calls_per_service[i]++;
+					s->calls++;
+				} else {
+					cs->calls_per_service[i]++;
+					s->spec.callback(userdata);
+					s->calls++;
+				}
+
+				rte_atomic32_clear(&s->execute_lock);
+			}
+		}
+	}
+
+	lcore_config[lcore].state = WAIT;
+
+	return 0;
+}
+
+int32_t
+rte_service_lcore_count(void)
+{
+	int32_t count = 0;
+	uint32_t i;
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		count += cores_state[i].is_service_core;
+	return count;
+}
+
+int32_t
+rte_service_lcore_list(uint32_t array[], uint32_t n)
+{
+	uint32_t count = rte_service_lcore_count();
+	if (count > n)
+		return -ENOMEM;
+
+	if (!array)
+		return -EINVAL;
+
+	uint32_t i;
+	uint32_t idx = 0;
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct core_state *cs = &cores_state[i];
+		if (cs->is_service_core) {
+			array[idx] = i;
+			idx++;
+		}
+	}
+
+	return count;
+}
+
+int32_t
+rte_service_set_default_mapping(void)
+{
+	/* create a default mapping from cores to services, then start the
+	 * services to make them transparent to unaware applications.
+	 */
+	uint32_t i;
+	int ret;
+	uint32_t count = rte_service_get_count();
+
+	int32_t lcore_iter = 0;
+	uint32_t ids[RTE_MAX_LCORE];
+	int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE);
+
+	for (i = 0; i < count; i++) {
+		struct rte_service_spec *s = rte_service_get_by_id(i);
+		if (!s)
+			return -EINVAL;
+
+		/* if no lcores available as services cores, don't setup map.
+		 * This means app logic must add cores, and setup mappings
+		 */
+		if (lcore_count > 0) {
+			/* do 1:1 core mapping here, with each service getting
+			 * assigned a single core by default. Adding multiple
+			 * services should multiplex to a single core, or 1:1
+			 * if services == cores
+			 */
+			ret = rte_service_enable_on_lcore(s, ids[lcore_iter]);
+			if (ret)
+				return -ENODEV;
+		}
+
+		lcore_iter++;
+		if (lcore_iter >= lcore_count)
+			lcore_iter = 0;
+
+		ret = rte_service_start(s);
+		if (ret)
+			return -ENOEXEC;
+	}
+
+	return 0;
+}
+
+static int32_t
+service_update(struct rte_service_spec *service, uint32_t lcore,
+		uint32_t *set, uint32_t *enabled)
+{
+	uint32_t i;
+	int32_t sid = -1;
+
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if ((struct rte_service_spec *)&rte_services[i] == service &&
+				service_valid(i)) {
+			sid = i;
+			break;
+		}
+	}
+
+	if (sid == -1 || lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	if (!cores_state[lcore].is_service_core)
+		return -EINVAL;
+
+	if (set) {
+		if (*set) {
+			cores_state[lcore].service_mask |=  (1 << sid);
+			rte_services[sid].num_mapped_cores++;
+		} else {
+			cores_state[lcore].service_mask &= ~(1 << sid);
+			rte_services[sid].num_mapped_cores--;
+		}
+	}
+
+	if (enabled)
+		*enabled = (cores_state[lcore].service_mask & (1 << sid));
+
+	rte_smp_wmb();
+
+	return 0;
+}
+
+int32_t rte_service_get_enabled_on_lcore(struct rte_service_spec *service,
+					uint32_t lcore)
+{
+	uint32_t enabled;
+	int ret = service_update(service, lcore, 0, &enabled);
+	if (ret == 0)
+		return enabled;
+	return -EINVAL;
+}
+
+int32_t
+rte_service_enable_on_lcore(struct rte_service_spec *service, uint32_t lcore)
+{
+	uint32_t on = 1;
+	return service_update(service, lcore, &on, 0);
+}
+
+int32_t
+rte_service_disable_on_lcore(struct rte_service_spec *service, uint32_t lcore)
+{
+	uint32_t off = 0;
+	return service_update(service, lcore, &off, 0);
+}
+
+int32_t rte_service_lcore_reset_all(void)
+{
+	/* loop over cores, reset all to mask 0 */
+	uint32_t i;
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		cores_state[i].service_mask = 0;
+		cores_state[i].is_service_core = 0;
+		cores_state[i].runstate = RUNSTATE_STOPPED;
+	}
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++)
+		rte_services[i].num_mapped_cores = 0;
+
+	rte_smp_wmb();
+
+	return 0;
+}
+
+int32_t
+rte_service_lcore_add(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+	if (cores_state[lcore].is_service_core)
+		return -EALREADY;
+
+	lcore_config[lcore].core_role = ROLE_SERVICE;
+
+	cores_state[lcore].is_service_core = 1;
+	cores_state[lcore].service_mask = 0;
+	cores_state[lcore].runstate = RUNSTATE_STOPPED;
+
+	return 0;
+}
+
+int32_t
+rte_service_lcore_del(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	struct core_state *cs = &cores_state[lcore];
+	if (!cs->is_service_core)
+		return -EINVAL;
+
+	if (cs->runstate != RUNSTATE_STOPPED)
+		return -EBUSY;
+
+	lcore_config[lcore].core_role = ROLE_RTE;
+	cores_state[lcore].is_service_core = 0;
+
+	return 0;
+}
+
+int32_t
+rte_service_lcore_start(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	struct core_state *cs = &cores_state[lcore];
+	if (!cs->is_service_core)
+		return -EINVAL;
+
+	if (cs->runstate == RUNSTATE_RUNNING)
+		return -EALREADY;
+
+	/* set core to run state first, and then launch otherwise it will
+	 * return immediately as runstate keeps it in the service poll loop
+	 */
+	cores_state[lcore].runstate = RUNSTATE_RUNNING;
+
+	int ret = rte_eal_remote_launch(rte_service_runner_func, 0, lcore);
+	/* returns -EBUSY if the core is already launched, 0 on success */
+	return ret;
+}
+
+int32_t
+rte_service_lcore_stop(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	if (cores_state[lcore].runstate == RUNSTATE_STOPPED)
+		return -EALREADY;
+
+	uint32_t i;
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		int32_t enabled = cores_state[i].service_mask & (1 << i);
+		int32_t service_running = rte_services[i].runstate !=
+						RUNSTATE_STOPPED;
+		int32_t only_core = rte_services[i].num_mapped_cores == 1;
+
+		/* if the core is mapped, and the service is running, and this
+		 * is the only core that is mapped, the service would cease to
+		 * run if this core stopped, so fail instead.
+		 */
+		if (enabled && service_running && only_core)
+			return -EBUSY;
+	}
+
+	cores_state[lcore].runstate = RUNSTATE_STOPPED;
+
+	return 0;
+}
+
+static void
+rte_service_dump_one(FILE *f, struct rte_service_spec_impl *s,
+		     uint64_t all_cycles, uint32_t reset)
+{
+	/* avoid divide by zero */
+	if (all_cycles == 0)
+		all_cycles = 1;
+
+	int calls = 1;
+	if (s->calls != 0)
+		calls = s->calls;
+
+	float cycles_pct = (((float)s->cycles_spent) / all_cycles) * 100.f;
+	fprintf(f,
+			"  %s : %0.1f %%\tcalls %"PRIu64"\tcycles %"
+			PRIu64"\tavg: %"PRIu64"\n",
+			s->spec.name, cycles_pct, s->calls, s->cycles_spent,
+			s->cycles_spent / calls);
+
+	if (reset) {
+		s->cycles_spent = 0;
+		s->calls = 0;
+	}
+}
+
+static void
+service_dump_calls_per_lcore(FILE *f, uint32_t lcore, uint32_t reset)
+{
+	uint32_t i;
+	struct core_state *cs = &cores_state[lcore];
+
+	fprintf(f, "%02d\t", lcore);
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (!service_valid(i))
+			continue;
+		fprintf(f, "%"PRIu64"\t", cs->calls_per_service[i]);
+		if (reset)
+			cs->calls_per_service[i] = 0;
+	}
+	fprintf(f, "\n");
+}
+
+int32_t rte_service_dump(FILE *f, struct rte_service_spec *service)
+{
+	uint32_t i;
+
+	uint64_t total_cycles = 0;
+	for (i = 0; i < rte_service_count; i++) {
+		if (!service_valid(i))
+			continue;
+		total_cycles += rte_services[i].cycles_spent;
+	}
+
+	int print_no_collect_warning = 0;
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		if (cores_state[i].collect_statistics == 0)
+			print_no_collect_warning = 1;
+	if (print_no_collect_warning)
+		fprintf(f, "Warning; cycle counts not collectd; refer to rte_service_set_stats_enable\n");
+
+	if (service) {
+		struct rte_service_spec_impl *s =
+			(struct rte_service_spec_impl *)service;
+		fprintf(f, "Service %s Summary\n", s->spec.name);
+		uint32_t reset = 0;
+		rte_service_dump_one(f, s, total_cycles, reset);
+		return 0;
+	}
+
+	fprintf(f, "Services Summary\n");
+	for (i = 0; i < rte_service_count; i++) {
+		uint32_t reset = 1;
+		rte_service_dump_one(f, &rte_services[i], total_cycles, reset);
+	}
+
+	fprintf(f, "Service Cores Summary\n");
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		if (lcore_config[i].core_role != ROLE_SERVICE)
+			continue;
+
+		uint32_t reset = 0;
+		service_dump_calls_per_lcore(f, i, reset);
+	}
+
+	return 0;
+}
diff --git a/lib/librte_eal/linuxapp/eal/Makefile b/lib/librte_eal/linuxapp/eal/Makefile
index 640afd0..438dcf9 100644
--- a/lib/librte_eal/linuxapp/eal/Makefile
+++ b/lib/librte_eal/linuxapp/eal/Makefile
@@ -96,6 +96,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_malloc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_elem.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_heap.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_keepalive.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_service.c
 
 # from arch dir
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_cpuflags.c
diff --git a/lib/librte_eal/linuxapp/eal/eal_thread.c b/lib/librte_eal/linuxapp/eal/eal_thread.c
index 9f88530..831ba07 100644
--- a/lib/librte_eal/linuxapp/eal/eal_thread.c
+++ b/lib/librte_eal/linuxapp/eal/eal_thread.c
@@ -184,7 +184,14 @@ eal_thread_loop(__attribute__((unused)) void *arg)
 		ret = lcore_config[lcore_id].f(fct_arg);
 		lcore_config[lcore_id].ret = ret;
 		rte_wmb();
-		lcore_config[lcore_id].state = FINISHED;
+
+		/* when a service core returns, it should go directly to WAIT
+		 * state, because the application will not lcore_wait() for it.
+		 */
+		if (lcore_config[lcore_id].core_role == ROLE_SERVICE)
+			lcore_config[lcore_id].state = WAIT;
+		else
+			lcore_config[lcore_id].state = FINISHED;
 	}
 
 	/* never reached */
diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
index 670bab3..830d224 100644
--- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
@@ -198,3 +198,32 @@ DPDK_17.05 {
 	vfio_get_group_no;
 
 } DPDK_17.02;
+
+DPDK_17.08 {
+	global:
+
+	rte_service_disable_on_lcore;
+	rte_service_dump;
+	rte_service_enable_on_lcore;
+	rte_service_get_by_id;
+	rte_service_get_by_name;
+	rte_service_get_count;
+	rte_service_get_enabled_on_lcore;
+	rte_service_is_running;
+	rte_service_lcore_add;
+	rte_service_lcore_count;
+	rte_service_lcore_del;
+	rte_service_lcore_list;
+	rte_service_lcore_reset_all;
+	rte_service_lcore_start;
+	rte_service_lcore_stop;
+	rte_service_probe_capability;
+	rte_service_register;
+	rte_service_reset;
+	rte_service_set_stats_enable;
+	rte_service_start;
+	rte_service_stop;
+	rte_service_unregister;
+
+
+} DPDK_17.05;
-- 
2.7.4

[PATCH v2 2/5] service cores: EAL init changes

[Harry van Haaren]

This commit shows the changes required in rte_eal_init()
to transparently launch the service threads. The threads
are launched into the service worker functions here because
after rte_eal_init() the application is not gauranteed to
call any other DPDK API.

As the registration of services happens at initialization
time, the services that require CPU time are already available
when we reach the end of rte_eal_init().

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

---

v2 comments:
- Include BSD implementation (Jerin)
- Move details of core-tracking into rte_service_lcore_add(Jerin)
- Given there are changes other to suggested, not using Ack
---
 lib/librte_eal/bsdapp/eal/eal.c   | 22 ++++++++++++++++++++++
 lib/librte_eal/linuxapp/eal/eal.c | 23 +++++++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c
index 05f0c1f..4f7dcb3 100644
--- a/lib/librte_eal/bsdapp/eal/eal.c
+++ b/lib/librte_eal/bsdapp/eal/eal.c
@@ -653,6 +653,17 @@ rte_eal_init(int argc, char **argv)
 	rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER);
 	rte_eal_mp_wait_lcore();
 
+	/* initialize services first so vdevs can register during bus_probe.
+	 * Ignore return value of already initialized, this means EAL parameter
+	 * -s was used to set a service-core mask.
+	 */
+	ret = rte_service_init();
+	if (ret) {
+		rte_eal_init_alert("rte_service_init() failed\n");
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	/* Probe all the buses and devices/drivers on them */
 	if (rte_bus_probe()) {
 		rte_eal_init_alert("Cannot probe devices\n");
@@ -660,6 +671,17 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 
+	/* initialize default services configuration */
+	uint32_t service_cores[RTE_MAX_LCORE];
+	int count = rte_service_lcore_list(service_cores, RTE_MAX_LCORE);
+	for (i = 0; i < count; i++)
+		rte_service_lcore_start(service_cores[i]);
+	ret = rte_service_set_default_mapping();
+	if (ret) {
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	rte_eal_mcfg_complete();
 
 	return fctret;
diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
index 7c78f2d..d63dd87 100644
--- a/lib/librte_eal/linuxapp/eal/eal.c
+++ b/lib/librte_eal/linuxapp/eal/eal.c
@@ -78,6 +78,7 @@
 #include <rte_version.h>
 #include <rte_atomic.h>
 #include <malloc_heap.h>
+#include <rte_service_private.h>
 
 #include "eal_private.h"
 #include "eal_thread.h"
@@ -932,6 +933,17 @@ rte_eal_init(int argc, char **argv)
 	rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER);
 	rte_eal_mp_wait_lcore();
 
+	/* initialize services first so vdevs can register during bus_probe.
+	 * Ignore return value of already initialized, this means EAL parameter
+	 * -s was used to set a service-core mask.
+	 */
+	ret = rte_service_init();
+	if (ret) {
+		rte_eal_init_alert("rte_service_init() failed\n");
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	/* Probe all the buses and devices/drivers on them */
 	if (rte_bus_probe()) {
 		rte_eal_init_alert("Cannot probe devices\n");
@@ -939,6 +951,17 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 
+	/* initialize default services configuration */
+	uint32_t service_cores[RTE_MAX_LCORE];
+	int count = rte_service_lcore_list(service_cores, RTE_MAX_LCORE);
+	for (i = 0; i < count; i++)
+		rte_service_lcore_start(service_cores[i]);
+	ret = rte_service_set_default_mapping();
+	if (ret) {
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	rte_eal_mcfg_complete();
 
 	return fctret;
-- 
2.7.4

[PATCH v2 3/5] service cores: coremask parsing

[Harry van Haaren]

Add logic for parsing a coremask from EAL, which allows
the application to be unaware of the cores being taken from
its coremask.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

---

v2:
- Remove printf() (Jerin)
- Remove commented code (Jerin)
- simplified core tracking, no requirement on
  #include rte_service in EAL parsing anymore.
---
 lib/librte_eal/common/eal_common_options.c | 77 ++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)

diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index f470195..cee200c 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -61,6 +61,7 @@ const char
 eal_short_options[] =
 	"b:" /* pci-blacklist */
 	"c:" /* coremask */
+	"s:" /* service coremask */
 	"d:" /* driver */
 	"h"  /* help */
 	"l:" /* corelist */
@@ -267,6 +268,73 @@ static int xdigit2val(unsigned char c)
 }
 
 static int
+eal_parse_service_coremask(const char *coremask)
+{
+	struct rte_config *cfg = rte_eal_get_configuration();
+	int i, j, idx = 0;
+	unsigned int count = 0;
+	char c;
+	int val;
+
+	if (coremask == NULL)
+		return -1;
+	/* Remove all blank characters ahead and after .
+	 * Remove 0x/0X if exists.
+	 */
+	while (isblank(*coremask))
+		coremask++;
+	if (coremask[0] == '0' && ((coremask[1] == 'x')
+		|| (coremask[1] == 'X')))
+		coremask += 2;
+	i = strlen(coremask);
+	while ((i > 0) && isblank(coremask[i - 1]))
+		i--;
+
+	if (i == 0)
+		return -1;
+
+	for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
+		c = coremask[i];
+		if (isxdigit(c) == 0) {
+			/* invalid characters */
+			return -1;
+		}
+		val = xdigit2val(c);
+		for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE;
+				j++, idx++) {
+			if ((1 << j) & val) {
+				/* handle master lcore already parsed */
+				uint32_t lcore = idx;
+				if (master_lcore_parsed &&
+						cfg->master_lcore == lcore)
+					continue;
+
+				if (!lcore_config[idx].detected) {
+					RTE_LOG(ERR, EAL,
+						"lcore %u unavailable\n", idx);
+					return -1;
+				}
+				lcore_config[idx].core_role = ROLE_SERVICE;
+				count++;
+			}
+		}
+	}
+
+	for (; i >= 0; i--)
+		if (coremask[i] != '0')
+			return -1;
+
+	for (; idx < RTE_MAX_LCORE; idx++)
+		lcore_config[idx].core_index = -1;
+
+	if (count == 0)
+		return -1;
+
+	cfg->service_lcore_count = count;
+	return 0;
+}
+
+static int
 eal_parse_coremask(const char *coremask)
 {
 	struct rte_config *cfg = rte_eal_get_configuration();
@@ -409,6 +477,8 @@ eal_parse_master_lcore(const char *arg)
 	if (cfg->master_lcore >= RTE_MAX_LCORE)
 		return -1;
 	master_lcore_parsed = 1;
+	/* ensure master core is not used as service core */
+	lcore_config[cfg->master_lcore].core_role = ROLE_RTE;
 	return 0;
 }
 
@@ -826,6 +896,13 @@ eal_parse_common_option(int opt, const char *optarg,
 		}
 		core_parsed = 1;
 		break;
+	/* service coremask */
+	case 's':
+		if (eal_parse_service_coremask(optarg) < 0) {
+			RTE_LOG(ERR, EAL, "invalid service coremask\n");
+			return -1;
+		}
+		break;
 	/* size of memory */
 	case 'm':
 		conf->memory = atoi(optarg);
-- 
2.7.4

[PATCH v2 4/5] service cores: add unit tests

[Harry van Haaren]

Add a bunch of unit tests, to ensure that the service
core functions are operating as expected.

As part of these tests a dummy service is registered which
allows identifying if a service callback has been invoked
by using the CPU tick counter. This allows identifying if
functions to start and stop service lcores are actually having
effect.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

---

v2 changes;
- Rename variable to slcore_id (Jerin)
- Rename function to unregister_all() (Jerin)
- Fix typos (Jerin)
- Add unit test for get_by_name()
- Add unit tests (all suggestions by Jerin)
-- get_name()
-- Verify probe_capability API
-- Verify MT_SAFE capability (see code for details)
-- Verify rte_service_dump() API
---
 test/test/Makefile             |   2 +
 test/test/test_service_cores.c | 496 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 498 insertions(+)
 create mode 100644 test/test/test_service_cores.c

diff --git a/test/test/Makefile b/test/test/Makefile
index ee240be..61e296b 100644
--- a/test/test/Makefile
+++ b/test/test/Makefile
@@ -151,6 +151,8 @@ SRCS-y += test_interrupts.c
 SRCS-y += test_version.c
 SRCS-y += test_func_reentrancy.c
 
+SRCS-y += test_service_cores.c
+
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline.c
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_num.c
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_etheraddr.c
diff --git a/test/test/test_service_cores.c b/test/test/test_service_cores.c
new file mode 100644
index 0000000..f2f3a93
--- /dev/null
+++ b/test/test/test_service_cores.c
@@ -0,0 +1,496 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <rte_common.h>
+#include <rte_hexdump.h>
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_memcpy.h>
+#include <rte_cycles.h>
+
+#include <rte_service.h>
+#include <rte_service_private.h>
+
+#include "test.h"
+
+/* used as the service core ID */
+static uint32_t slcore_id;
+/* used as timestamp to detect if a service core is running */
+static uint64_t service_tick;
+
+#define SERVICE_DELAY 1
+
+#define DUMMY_SERVICE_NAME "dummy_service"
+#define MT_SAFE_SERVICE_NAME "mt_safe_service"
+
+static int
+testsuite_setup(void)
+{
+	/* assuming lcore 1 is available for service-core testing */
+	slcore_id = 1;
+	return TEST_SUCCESS;
+}
+
+static void
+testsuite_teardown(void)
+{
+	/* release service cores? */
+}
+
+static int32_t dummy_cb(void *args)
+{
+	RTE_SET_USED(args);
+	service_tick++;
+	rte_delay_ms(SERVICE_DELAY);
+	return 0;
+}
+
+
+static int32_t dummy_mt_safe_cb(void *args)
+{
+	/* Atomic checks to ensure MT safe services allow > 1 thread to
+	 * concurrently run the callback. The concept is as follows;
+	 * 1) if lock is available, take the lock then delay
+	 * 2) if first lock is taken, and a thread arrives in the CB, we know
+	 *    that 2 threads are running the callback at the same time: MT safe
+	 */
+	uint32_t *test_params = args;
+	uint32_t *atomic_lock = &test_params[0];
+	uint32_t *pass_test = &test_params[1];
+	int lock_taken = rte_atomic32_cmpset(atomic_lock, 0, 1);
+	if (lock_taken) {
+		/* delay with the lock held */
+		rte_delay_ms(250);
+		rte_atomic32_clear((rte_atomic32_t *)atomic_lock);
+	} else {
+		/* 2nd thread will fail to take lock, so set pass flag */
+		*pass_test = 1;
+	}
+
+	return 0;
+}
+
+/* unregister all services */
+static int
+unregister_all(void)
+{
+	uint32_t i;
+	struct rte_service_spec *dead = (struct rte_service_spec *)0xdead;
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_unregister(0),
+			"Unregistered NULL pointer");
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_unregister(dead),
+			"Unregistered invalid pointer");
+
+	uint32_t c = rte_service_get_count();
+	for (i = 0; i < c; i++) {
+		struct rte_service_spec *s = rte_service_get_by_id(i);
+		TEST_ASSERT_EQUAL(0, rte_service_unregister(s),
+				"Error unregistering a valid service");
+	}
+
+	rte_service_lcore_reset_all();
+
+	return TEST_SUCCESS;
+}
+
+/* register a single dummy service */
+static int
+dummy_register(void)
+{
+	/* make sure there are no remains from previous tests */
+	unregister_all();
+
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid callback");
+	service.callback = dummy_cb;
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid name");
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Failed to register valid service");
+
+	return TEST_SUCCESS;
+}
+
+/* verify get_by_name() service lookup */
+static int
+service_get_by_name(void)
+{
+	unregister_all();
+
+	/* ensure with no services registered returns NULL */
+	TEST_ASSERT_EQUAL(0, rte_service_get_by_name(DUMMY_SERVICE_NAME),
+			"Service get by name should return NULL");
+
+	/* register service */
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid callback");
+	service.callback = dummy_cb;
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid name");
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Failed to register valid service");
+
+	/* ensure with dummy services registered returns same ptr as ID */
+	struct rte_service_spec *s_by_id = rte_service_get_by_id(0);
+	TEST_ASSERT_EQUAL(s_by_id, rte_service_get_by_name(DUMMY_SERVICE_NAME),
+			"Service get_by_name should equal get_by_id()");
+
+	unregister_all();
+
+	/* ensure after unregister, get_by_name returns NULL */
+	TEST_ASSERT_EQUAL(0, rte_service_get_by_name(DUMMY_SERVICE_NAME),
+			"get by name should return NULL after unregister");
+
+	return TEST_SUCCESS;
+}
+
+/* verify probe of capabilities */
+static int
+service_probe_capability(void)
+{
+	unregister_all();
+
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	service.callback = dummy_cb;
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+	service.capabilities |= RTE_SERVICE_CAP_MT_SAFE;
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Register of MT SAFE service failed");
+
+	/* verify flag is enabled */
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+	int32_t mt = rte_service_probe_capability(s, RTE_SERVICE_CAP_MT_SAFE);
+	TEST_ASSERT_EQUAL(1, mt, "MT SAFE capability flag not set.");
+
+
+	unregister_all();
+
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	service.callback = dummy_cb;
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Register of non-MT safe service failed");
+
+	/* verify flag is enabled */
+	s = rte_service_get_by_id(0);
+	mt = rte_service_probe_capability(s, RTE_SERVICE_CAP_MT_SAFE);
+	TEST_ASSERT_EQUAL(0, mt, "MT SAFE cap flag set on non MT SAFE service");
+
+	return unregister_all();
+}
+
+/* verify the service name */
+static int
+service_name(void)
+{
+	struct rte_service_spec *service = rte_service_get_by_id(0);
+
+	int equal = strcmp(service->name, DUMMY_SERVICE_NAME);
+	TEST_ASSERT_EQUAL(0, equal, "Error: Service name not correct");
+
+	return unregister_all();
+}
+
+/* verify service dump */
+static int
+service_dump(void)
+{
+	struct rte_service_spec *service = rte_service_get_by_id(0);
+	rte_service_set_stats_enable(1);
+	rte_service_dump(stdout, service);
+	return unregister_all();
+}
+
+/* start and stop a service */
+static int
+service_start_stop(void)
+{
+	struct rte_service_spec *service = rte_service_get_by_id(0);
+
+	/* is_running() returns if service is running and slcore is mapped */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(1),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(service, 1),
+			"Enabling service core");
+
+	TEST_ASSERT_EQUAL(0, rte_service_is_running(service),
+			"Error: Service should be stopped");
+
+	TEST_ASSERT_EQUAL(0, rte_service_stop(service),
+			"Error: Service stopped returned non-zero");
+
+	TEST_ASSERT_EQUAL(0, rte_service_is_running(service),
+			"Error: Service is running - should be stopped");
+
+	TEST_ASSERT_EQUAL(0, rte_service_start(service),
+			"Error: Service start returned non-zero");
+
+	TEST_ASSERT_EQUAL(1, rte_service_is_running(service),
+			"Error: Service is not running");
+
+	return unregister_all();
+}
+
+/* enable and disable a lcore for a service */
+static int
+service_lcore_en_dis_able(void)
+{
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+
+	/* expected failure cases */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_enable_on_lcore(s, 100000),
+			"Enable on invalid core did not fail");
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_disable_on_lcore(s, 100000),
+			"Disable on invalid core did not fail");
+
+	/* add service core to allow enabling */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Add service core failed when not in use before");
+
+	/* valid enable */
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, slcore_id),
+			"Enabling valid service and core failed");
+	TEST_ASSERT_EQUAL(1, rte_service_get_enabled_on_lcore(s, slcore_id),
+			"Enabled core returned not-enabled");
+
+	/* valid disable */
+	TEST_ASSERT_EQUAL(0, rte_service_disable_on_lcore(s, slcore_id),
+			"Disabling valid service and lcore failed");
+	TEST_ASSERT_EQUAL(0, rte_service_get_enabled_on_lcore(s, slcore_id),
+			"Disabled core returned enabled");
+
+	return unregister_all();
+}
+
+static int
+service_lcore_running_check(void)
+{
+	uint64_t tick = service_tick;
+	rte_delay_ms(SERVICE_DELAY * 10);
+	/* if (tick != service_tick) we know the lcore as polled the service */
+	return tick != service_tick;
+}
+
+static int
+service_lcore_add_del(void)
+{
+	/* check initial count */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_count(),
+			"Service lcore count has value before adding a lcore");
+
+	/* check service lcore add */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Add service core failed when not in use before");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_add(slcore_id),
+			"Add service core failed to refuse in-use lcore");
+
+	/* check count */
+	TEST_ASSERT_EQUAL(1, rte_service_lcore_count(),
+			"Service core count not equal to one");
+
+	/* retrieve core list, checking lcore ids */
+	const uint32_t size = 4;
+	uint32_t service_core_ids[size];
+	int32_t n = rte_service_lcore_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(1, n, "Service core list return should equal 1");
+	TEST_ASSERT_EQUAL(slcore_id, service_core_ids[0],
+				"Service core list lcore must equal slcore_id");
+
+	/* recheck count, add more cores, and check count */
+	TEST_ASSERT_EQUAL(1, rte_service_lcore_count(),
+			"Service core count not equal to one");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(2),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(3),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(3, rte_service_lcore_count(),
+			"Service core count not equal to three");
+
+	/* check longer service core list */
+	n = rte_service_lcore_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(3, n, "Service core list return should equal 3");
+	TEST_ASSERT_EQUAL(slcore_id, service_core_ids[0],
+				"Service core list[0] lcore must equal 1");
+	TEST_ASSERT_EQUAL(2, service_core_ids[1],
+				"Service core list[1] lcore must equal 2");
+	TEST_ASSERT_EQUAL(3, service_core_ids[2],
+				"Service core list[2] lcore must equal 3");
+
+	/* recheck count, remove lcores, check remaining lcore_id is correct */
+	TEST_ASSERT_EQUAL(3, rte_service_lcore_count(),
+			"Service core count not equal to three");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_del(1),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_del(2),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(1, rte_service_lcore_count(),
+			"Service core count not equal to one");
+	n = rte_service_lcore_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(1, n, "Service core list return should equal one");
+	TEST_ASSERT_EQUAL(3, service_core_ids[0],
+				"Service core list[0] lcore must equal three");
+
+	return unregister_all();
+}
+
+static int
+service_mt_safe_poll(void)
+{
+	unregister_all();
+
+	/* add 2 cores */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(1), "mt safe lcore add fail");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(2), "mt safe lcore add fail");
+
+	/* Use atomic locks to verify that two threads are in the same function
+	 * at the same time. These are passed to the unit tests through the
+	 * callback userdata parameter
+	 */
+	uint32_t test_params[2];
+	memset(test_params, 0, sizeof(uint32_t) * 2);
+
+	/* register MT safe service. */
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	service.callback = dummy_mt_safe_cb;
+	service.callback_userdata = test_params;
+	snprintf(service.name, sizeof(service.name), MT_SAFE_SERVICE_NAME);
+	service.capabilities |= RTE_SERVICE_CAP_MT_SAFE;
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Register of MT SAFE service failed");
+
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+	TEST_ASSERT_EQUAL(0, rte_service_start(s),
+			"Starting valid service failed");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, 1),
+			"Failed to enable lcore 1 on mt safe service");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, 2),
+			"Failed to enable lcore 2 on mt safe service");
+	rte_service_lcore_start(1);
+	rte_service_lcore_start(2);
+
+	/* wait for the worker threads to run */
+	rte_delay_ms(500);
+	rte_service_lcore_stop(1);
+	rte_service_lcore_stop(2);
+
+	TEST_ASSERT_EQUAL(1, test_params[1],
+			"MT Safe service not run by two cores concurrently");
+
+	TEST_ASSERT_EQUAL(0, rte_service_stop(s),
+			"Failed to stop MT Safe service");
+
+	return unregister_all();
+}
+
+/* start and stop a service core - ensuring it goes back to sleep */
+static int
+service_lcore_start_stop(void)
+{
+	/* start service core and service, create mapping so tick() runs */
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+	TEST_ASSERT_EQUAL(0, rte_service_start(s),
+			"Starting valid service failed");
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_enable_on_lcore(s, slcore_id),
+			"Enabling valid service on non-service core must fail");
+
+	/* core start */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_lcore_start(slcore_id),
+			"Service core start without add should return EINVAL");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, slcore_id),
+			"Enabling valid service on valid core failed");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_start(slcore_id),
+			"Service core start after add failed");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_start(slcore_id),
+			"Service core expected as running but was stopped");
+
+	/* ensures core really is running the service function */
+	TEST_ASSERT_EQUAL(1, service_lcore_running_check(),
+			"Service core expected to poll service but it didn't");
+
+	/* core stop */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_lcore_stop(100000),
+			"Invalid Service core stop should return -EINVAL");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_stop(slcore_id),
+			"Service core stop expected to return 0");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_stop(slcore_id),
+			"Already stopped service core should return -EALREADY");
+
+	/* ensure service is not longer running */
+	TEST_ASSERT_EQUAL(0, service_lcore_running_check(),
+			"Service core expected to poll service but it didn't");
+
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_del(slcore_id),
+			"Service core del did not return zero");
+
+	return unregister_all();
+}
+
+static struct unit_test_suite service_tests  = {
+	.suite_name = "service core test suite",
+	.setup = testsuite_setup,
+	.teardown = testsuite_teardown,
+	.unit_test_cases = {
+		TEST_CASE_ST(dummy_register, NULL, unregister_all),
+		TEST_CASE_ST(dummy_register, NULL, service_name),
+		TEST_CASE_ST(dummy_register, NULL, service_get_by_name),
+		TEST_CASE_ST(dummy_register, NULL, service_dump),
+		TEST_CASE_ST(dummy_register, NULL, service_probe_capability),
+		TEST_CASE_ST(dummy_register, NULL, service_start_stop),
+		TEST_CASE_ST(dummy_register, NULL, service_lcore_add_del),
+		TEST_CASE_ST(dummy_register, NULL, service_lcore_start_stop),
+		TEST_CASE_ST(dummy_register, NULL, service_lcore_en_dis_able),
+		TEST_CASE_ST(dummy_register, NULL, service_mt_safe_poll),
+		TEST_CASES_END() /**< NULL terminate unit test array */
+	}
+};
+
+static int
+test_service_common(void)
+{
+	return unit_test_suite_runner(&service_tests);
+}
+
+REGISTER_TEST_COMMAND(service_autotest, test_service_common);
-- 
2.7.4

[PATCH v2 5/5] service cores: enable event/sw with service

[Harry van Haaren]

This commit shows how easy it is to enable a specific
DPDK component with a service callback, in order to get
CPU cycles for it.

The beauty of this method is that the service is unaware
of how much CPU time it is getting - the application can
decide how to split and slice cores and map them to the
registered services.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

---

v2:
- Remove #include <rte_cycles.h> (Jerin)
- Remove development prints (Jerin)
- Track service name in PMD
- Print warning if service does not have an lcore mapped (Jerin)
---
 drivers/event/sw/sw_evdev.c | 32 ++++++++++++++++++++++++++++++++
 drivers/event/sw/sw_evdev.h |  3 +++
 2 files changed, 35 insertions(+)

diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
index fe2a61e..baab376 100644
--- a/drivers/event/sw/sw_evdev.c
+++ b/drivers/event/sw/sw_evdev.c
@@ -38,6 +38,7 @@
 #include <rte_kvargs.h>
 #include <rte_ring.h>
 #include <rte_errno.h>
+#include <rte_service_private.h>
 
 #include "sw_evdev.h"
 #include "iq_ring.h"
@@ -597,6 +598,13 @@ sw_start(struct rte_eventdev *dev)
 {
 	unsigned int i, j;
 	struct sw_evdev *sw = sw_pmd_priv(dev);
+
+	/* check a service core is mapped to this service */
+	struct rte_service_spec *s = rte_service_get_by_name(sw->service_name);
+	if (!rte_service_is_running(s))
+		SW_LOG_ERR("Warning: No Service core enabled on service %s\n",
+				s->name);
+
 	/* check all ports are set up */
 	for (i = 0; i < sw->port_count; i++)
 		if (sw->ports[i].rx_worker_ring == NULL) {
@@ -699,6 +707,14 @@ set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque)
 	return 0;
 }
 
+
+static int32_t sw_sched_service_func(void *args)
+{
+	struct rte_eventdev *dev = args;
+	sw_event_schedule(dev);
+	return 0;
+}
+
 static int
 sw_probe(struct rte_vdev_device *vdev)
 {
@@ -810,6 +826,22 @@ sw_probe(struct rte_vdev_device *vdev)
 	sw->credit_update_quanta = credit_quanta;
 	sw->sched_quanta = sched_quanta;
 
+	/* register service with EAL */
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	snprintf(service.name, sizeof(service.name), "%s_service", name);
+	snprintf(sw->service_name, sizeof(sw->service_name), "%s_service",
+			name);
+	service.socket_id = socket_id;
+	service.callback = sw_sched_service_func;
+	service.callback_userdata = (void *)dev;
+
+	int32_t ret = rte_service_register(&service);
+	if (ret) {
+		SW_LOG_ERR("service register() failed");
+		return -ENOEXEC;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/event/sw/sw_evdev.h b/drivers/event/sw/sw_evdev.h
index 0d7f94f..3e83823 100644
--- a/drivers/event/sw/sw_evdev.h
+++ b/drivers/event/sw/sw_evdev.h
@@ -59,6 +59,7 @@
 
 #define EVENTDEV_NAME_SW_PMD event_sw
 #define SW_PMD_NAME RTE_STR(event_sw)
+#define SW_PMD_NAME_MAX 64
 
 #define SW_SCHED_TYPE_DIRECT (RTE_SCHED_TYPE_PARALLEL + 1)
 
@@ -276,6 +277,8 @@ struct sw_evdev {
 	/* store num stats and offset of the stats for each queue */
 	uint16_t xstats_count_per_qid[RTE_EVENT_MAX_QUEUES_PER_DEV];
 	uint16_t xstats_offset_for_qid[RTE_EVENT_MAX_QUEUES_PER_DEV];
+
+	char service_name[SW_PMD_NAME_MAX];
 };
 
 static inline struct sw_evdev *
-- 
2.7.4

[PATCH v3 0/7] service cores: cover letter

[Harry van Haaren]

This patchset introduces service cores to DPDK. A service core
is an lcore that performs functions to abstract away details of
differences in environment of the application.

An example is using the eventdev API, where either a software or hardware
PMD performs scheduling. In the case of the software PMD an lcore is
required to perform scheduling, which means application logic would have
to be aware of the PMD running under the API. To abstract away the
differences in HW / SW PMDs, service cores can run the SW PMD service
without application logic specifying the exact cores to use. Note that
eventdev is only one API that benefits; timers, interrupts handling,
statistics and monitoring, and a range of other infrastructure that
requires a slice of CPU time may all benefit from service cores.

The application is not obliged to manually use the service cores API,
however if an application wishes to use the service cores API for fine
grained control over how the services are run, this is possible. Deciding
between a performance threading-profile and scaled-down profile can be
achieved by advanced usage of service cores and setting the lcore mappings.

Finally, the last patch introduces how a PMD can register a service to run
a function. This is then available (along with any other registered services)
to be run by the service cores.

Regards, -Harry


v3:
- Added docs
- Added release notes
- Updated maintainers file
- Compile checks with devtools/test-build.sh
- Validated patches apply to latest dpdk/master
- Based on discussion, rte_service_iterate() is *not* included,
  but could be adding at a later date if use-cases require it.
- Future work includes enabling the eventdev_pipeline sample app, but there
  is still some churn there to enable both HW/SW PMDs seamlessly. Once sample
  app is enabled a service core walk-through with that sample app can be added
  to the docs, to provide a tutorial on service-core usage.


Harry van Haaren (7):
  service cores: header and implementation
  service cores: EAL init changes
  service cores: coremask parsing
  service cores: add unit tests
  service cores: enable event/sw with service
  maintainers: claim service cores
  doc: add service cores to doc and release notes

 MAINTAINERS                                        |   6 +
 doc/api/doxy-api-index.md                          |   1 +
 doc/guides/eventdevs/sw.rst                        |   4 +-
 doc/guides/prog_guide/index.rst                    |   1 +
 doc/guides/prog_guide/service_cores.rst            |  81 +++
 doc/guides/rel_notes/release_17_08.rst             |   8 +
 drivers/event/sw/sw_evdev.c                        |  32 +
 drivers/event/sw/sw_evdev.h                        |   3 +
 lib/librte_eal/bsdapp/eal/Makefile                 |   1 +
 lib/librte_eal/bsdapp/eal/eal.c                    |  22 +
 lib/librte_eal/bsdapp/eal/rte_eal_version.map      |  28 +
 lib/librte_eal/common/Makefile                     |   1 +
 lib/librte_eal/common/eal_common_lcore.c           |   1 +
 lib/librte_eal/common/eal_common_options.c         |  77 +++
 lib/librte_eal/common/include/rte_eal.h            |   4 +
 lib/librte_eal/common/include/rte_lcore.h          |   3 +-
 lib/librte_eal/common/include/rte_service.h        | 298 +++++++++
 .../common/include/rte_service_private.h           | 118 ++++
 lib/librte_eal/common/rte_service.c                | 671 +++++++++++++++++++++
 lib/librte_eal/linuxapp/eal/Makefile               |   1 +
 lib/librte_eal/linuxapp/eal/eal.c                  |  23 +
 lib/librte_eal/linuxapp/eal/eal_thread.c           |   9 +-
 lib/librte_eal/linuxapp/eal/rte_eal_version.map    |  29 +
 test/test/Makefile                                 |   2 +
 test/test/test_service_cores.c                     | 496 +++++++++++++++
 25 files changed, 1917 insertions(+), 3 deletions(-)
 create mode 100644 doc/guides/prog_guide/service_cores.rst
 create mode 100644 lib/librte_eal/common/include/rte_service.h
 create mode 100644 lib/librte_eal/common/include/rte_service_private.h
 create mode 100644 lib/librte_eal/common/rte_service.c
 create mode 100644 test/test/test_service_cores.c

-- 
2.7.4

[PATCH v3 1/7] service cores: header and implementation

[Harry van Haaren]

Add header files, update .map files with new service
functions, and add the service header to the doxygen
for building.

This service header API allows DPDK to use services as
a concept of something that requires CPU cycles. An example
is a PMD that runs in software to schedule events, where a
hardware version exists that does not require a CPU.

The code presented here is based on an initial RFC:
http://dpdk.org/ml/archives/dev/2017-May/065207.html
This was then reworked, and RFC v2 with the changes posted:
http://dpdk.org/ml/archives/dev/2017-June/067194.html

This is the fourth iteration of the service core concept,
with 2 RFCs and this being v2 of the implementation.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

---

v2:
Thanks Jerin for review - below a list your suggested changes;
- Doxygen rename to "service cores" for consistency
- use lcore instead of core for function names
- Fix about 10 typos / seplling msitakse ;)
- Dix doxygen /** comments for functions
- Doxygen @param[out] improvements
- int8_t for socket_id to ordinary int
- Rename MACROS for readability
- Align structs to cache lines
- Allocate fastpath-used data from hugepages
- Added/fixed memory barriers for multi-core scheduling
- Add const to variables, and hoist above loop
- Optimize cmpset atomic if MT_SAFE or only one core mapped
- Statistics collection only when requested
- Add error check for array pointer
- Remove panic() calls from library
- Fix TODO notes from previous patchset

There are also some other changes;
- Checkpatch issues fixed
- .map file updates
- Add rte_service_get_by_name() function
---
 doc/api/doxy-api-index.md                          |   1 +
 lib/librte_eal/bsdapp/eal/Makefile                 |   1 +
 lib/librte_eal/bsdapp/eal/rte_eal_version.map      |  28 +
 lib/librte_eal/common/Makefile                     |   1 +
 lib/librte_eal/common/eal_common_lcore.c           |   1 +
 lib/librte_eal/common/include/rte_eal.h            |   4 +
 lib/librte_eal/common/include/rte_lcore.h          |   3 +-
 lib/librte_eal/common/include/rte_service.h        | 298 +++++++++
 .../common/include/rte_service_private.h           | 118 ++++
 lib/librte_eal/common/rte_service.c                | 671 +++++++++++++++++++++
 lib/librte_eal/linuxapp/eal/Makefile               |   1 +
 lib/librte_eal/linuxapp/eal/eal_thread.c           |   9 +-
 lib/librte_eal/linuxapp/eal/rte_eal_version.map    |  29 +
 13 files changed, 1163 insertions(+), 2 deletions(-)
 create mode 100644 lib/librte_eal/common/include/rte_service.h
 create mode 100644 lib/librte_eal/common/include/rte_service_private.h
 create mode 100644 lib/librte_eal/common/rte_service.c

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index f5f1f19..1284402 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -158,6 +158,7 @@ There are many libraries, so their headers may be grouped by topics:
   [common]             (@ref rte_common.h),
   [ABI compat]         (@ref rte_compat.h),
   [keepalive]          (@ref rte_keepalive.h),
+  [service cores]      (@ref rte_service.h),
   [device metrics]     (@ref rte_metrics.h),
   [bitrate statistics] (@ref rte_bitrate.h),
   [latency statistics] (@ref rte_latencystats.h),
diff --git a/lib/librte_eal/bsdapp/eal/Makefile b/lib/librte_eal/bsdapp/eal/Makefile
index a0f9950..05517a2 100644
--- a/lib/librte_eal/bsdapp/eal/Makefile
+++ b/lib/librte_eal/bsdapp/eal/Makefile
@@ -87,6 +87,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_malloc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_elem.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_heap.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_keepalive.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_service.c
 
 # from arch dir
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_cpuflags.c
diff --git a/lib/librte_eal/bsdapp/eal/rte_eal_version.map b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
index 2e48a73..5493a13 100644
--- a/lib/librte_eal/bsdapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
@@ -193,3 +193,31 @@ DPDK_17.05 {
 	vfio_get_group_no;
 
 } DPDK_17.02;
+
+DPDK_17.08 {
+	global:
+
+	rte_service_disable_on_lcore;
+	rte_service_dump;
+	rte_service_enable_on_lcore;
+	rte_service_get_by_id;
+	rte_service_get_by_name;
+	rte_service_get_count;
+	rte_service_get_enabled_on_lcore;
+	rte_service_is_running;
+	rte_service_lcore_add;
+	rte_service_lcore_count;
+	rte_service_lcore_del;
+	rte_service_lcore_list;
+	rte_service_lcore_reset_all;
+	rte_service_lcore_start;
+	rte_service_lcore_stop;
+	rte_service_probe_capability;
+	rte_service_register;
+	rte_service_reset;
+	rte_service_set_stats_enable;
+	rte_service_start;
+	rte_service_stop;
+	rte_service_unregister;
+
+} DPDK_17.05;
diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
index a5bd108..2a93397 100644
--- a/lib/librte_eal/common/Makefile
+++ b/lib/librte_eal/common/Makefile
@@ -41,6 +41,7 @@ INC += rte_eal_memconfig.h rte_malloc_heap.h
 INC += rte_hexdump.h rte_devargs.h rte_bus.h rte_dev.h rte_vdev.h
 INC += rte_pci_dev_feature_defs.h rte_pci_dev_features.h
 INC += rte_malloc.h rte_keepalive.h rte_time.h
+INC += rte_service.h rte_service_private.h
 
 GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h
 GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h
diff --git a/lib/librte_eal/common/eal_common_lcore.c b/lib/librte_eal/common/eal_common_lcore.c
index 84fa0cb..0db1555 100644
--- a/lib/librte_eal/common/eal_common_lcore.c
+++ b/lib/librte_eal/common/eal_common_lcore.c
@@ -81,6 +81,7 @@ rte_eal_cpu_init(void)
 
 		/* By default, each detected core is enabled */
 		config->lcore_role[lcore_id] = ROLE_RTE;
+		lcore_config[lcore_id].core_role = ROLE_RTE;
 		lcore_config[lcore_id].core_id = eal_cpu_core_id(lcore_id);
 		lcore_config[lcore_id].socket_id = eal_cpu_socket_id(lcore_id);
 		if (lcore_config[lcore_id].socket_id >= RTE_MAX_NUMA_NODES) {
diff --git a/lib/librte_eal/common/include/rte_eal.h b/lib/librte_eal/common/include/rte_eal.h
index abf020b..4dd0518 100644
--- a/lib/librte_eal/common/include/rte_eal.h
+++ b/lib/librte_eal/common/include/rte_eal.h
@@ -61,6 +61,7 @@ extern "C" {
 enum rte_lcore_role_t {
 	ROLE_RTE,
 	ROLE_OFF,
+	ROLE_SERVICE,
 };
 
 /**
@@ -80,6 +81,7 @@ enum rte_proc_type_t {
 struct rte_config {
 	uint32_t master_lcore;       /**< Id of the master lcore */
 	uint32_t lcore_count;        /**< Number of available logical cores. */
+	uint32_t service_lcore_count;/**< Number of available service cores. */
 	enum rte_lcore_role_t lcore_role[RTE_MAX_LCORE]; /**< State of cores. */
 
 	/** Primary or secondary configuration */
@@ -185,6 +187,8 @@ int rte_eal_iopl_init(void);
  *
  *     EPROTO indicates that the PCI bus is either not present, or is not
  *            readable by the eal.
+ *
+ *     ENOEXEC indicates that a service core failed to launch successfully.
  */
 int rte_eal_init(int argc, char **argv);
 
diff --git a/lib/librte_eal/common/include/rte_lcore.h b/lib/librte_eal/common/include/rte_lcore.h
index fe7b586..50e0d0f 100644
--- a/lib/librte_eal/common/include/rte_lcore.h
+++ b/lib/librte_eal/common/include/rte_lcore.h
@@ -73,6 +73,7 @@ struct lcore_config {
 	unsigned core_id;          /**< core number on socket for this lcore */
 	int core_index;            /**< relative index, starting from 0 */
 	rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
+	uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
 };
 
 /**
@@ -175,7 +176,7 @@ rte_lcore_is_enabled(unsigned lcore_id)
 	struct rte_config *cfg = rte_eal_get_configuration();
 	if (lcore_id >= RTE_MAX_LCORE)
 		return 0;
-	return cfg->lcore_role[lcore_id] != ROLE_OFF;
+	return cfg->lcore_role[lcore_id] == ROLE_RTE;
 }
 
 /**
diff --git a/lib/librte_eal/common/include/rte_service.h b/lib/librte_eal/common/include/rte_service.h
new file mode 100644
index 0000000..3be59ea
--- /dev/null
+++ b/lib/librte_eal/common/include/rte_service.h
@@ -0,0 +1,298 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_SERVICE_H_
+#define _RTE_SERVICE_H_
+
+/**
+ * @file
+ *
+ * Service functions
+ *
+ * The service functionality provided by this header allows a DPDK component
+ * to indicate that it requires a function call in order for it to perform
+ * its processing.
+ *
+ * An example usage of this functionality would be a component that registers
+ * a service to perform a particular packet processing duty: for example the
+ * eventdev software PMD. At startup the application requests all services
+ * that have been registered, and the cores in the service-coremask run the
+ * required services. The EAL removes these number of cores from the available
+ * runtime cores, and dedicates them to performing service-core workloads. The
+ * application has access to the remaining lcores as normal.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include<stdio.h>
+#include <stdint.h>
+#include <sys/queue.h>
+
+#include <rte_lcore.h>
+
+/* forward declaration only. Definition in rte_service_private.h */
+struct rte_service_spec;
+
+#define RTE_SERVICE_NAME_MAX 32
+
+/* Capabilities of a service.
+ *
+ * Use the *rte_service_probe_capability* function to check if a service is
+ * capable of a specific capability.
+ */
+/** When set, the service is capable of having multiple threads run it at the
+ *  same time.
+ */
+#define RTE_SERVICE_CAP_MT_SAFE (1 << 0)
+
+/** Return the number of services registered.
+ *
+ * The number of services registered can be passed to *rte_service_get_by_id*,
+ * enabling the application to retrieve the specification of each service.
+ *
+ * @return The number of services registered.
+ */
+uint32_t rte_service_get_count(void);
+
+/** Return the specification of a service by integer id.
+ *
+ * This function provides the specification of a service. This can be used by
+ * the application to understand what the service represents. The service
+ * must not be modified by the application directly, only passed to the various
+ * rte_service_* functions.
+ *
+ * @param id The integer id of the service to retrieve
+ * @retval non-zero A valid pointer to the service_spec
+ * @retval NULL Invalid *id* provided.
+ */
+struct rte_service_spec *rte_service_get_by_id(uint32_t id);
+
+/** Return the specification of a service by name.
+ *
+ * This function provides the specification of a service using the service name
+ * as lookup key. This can be used by the application to understand what the
+ * service represents. The service must not be modified by the application
+ * directly, only passed to the various rte_service_* functions.
+ *
+ * @param name The name of the service to retrieve
+ * @retval non-zero A valid pointer to the service_spec
+ * @retval NULL Invalid *name* provided.
+ */
+struct rte_service_spec *rte_service_get_by_name(const char *name);
+
+/** Return the name of the service.
+ *
+ * @return A pointer to the name of the service. The returned pointer remains
+ *         in ownership of the service, and the application must not free it.
+ */
+const char *rte_service_get_name(const struct rte_service_spec *service);
+
+/** Check if a service has a specific capability.
+ *
+ * This function returns if *service* has implements *capability*.
+ * See RTE_SERVICE_CAP_* defines for a list of valid capabilities.
+ * @retval 1 Capability supported by this service instance
+ * @retval 0 Capability not supported by this service instance
+ */
+int32_t rte_service_probe_capability(const struct rte_service_spec *service,
+				     uint32_t capability);
+
+/** Enable a core to run a service.
+ *
+ * Each core can be added or removed from running specific services. This
+ * functions adds *lcore* to the set of cores that will run *service*.
+ *
+ * If multiple cores are enabled on a service, an atomic is used to ensure that
+ * only one cores runs the service at a time. The exception to this is when
+ * a service indicates that it is multi-thread safe by setting the capability
+ * called RTE_SERVICE_CAP_MT_SAFE. With the multi-thread safe capability set,
+ * the service function can be run on multiple threads at the same time.
+ *
+ * @retval 0 lcore added successfully
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_enable_on_lcore(struct rte_service_spec *service,
+				   uint32_t lcore);
+
+/** Disable a core to run a service.
+ *
+ * Each core can be added or removed from running specific services. This
+ * functions removes *lcore* to the set of cores that will run *service*.
+ *
+ * @retval 0 Lcore removed successfully
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_disable_on_lcore(struct rte_service_spec *service,
+				   uint32_t lcore);
+
+/** Return if an lcore is enabled for the service.
+ *
+ * This function allows the application to query if *lcore* is currently set to
+ * run *service*.
+ *
+ * @retval 1 Lcore enabled on this lcore
+ * @retval 0 Lcore disabled on this lcore
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_get_enabled_on_lcore(struct rte_service_spec *service,
+					uint32_t lcore);
+
+
+/** Enable *service* to run.
+ *
+ * This function switches on a service during runtime.
+ * @retval 0 The service was successfully started
+ */
+int32_t rte_service_start(struct rte_service_spec *service);
+
+/** Disable *service*.
+ *
+ * Switch off a service, so it is not run until it is *rte_service_start* is
+ * called on it.
+ * @retval 0 Service successfully switched off
+ */
+int32_t rte_service_stop(struct rte_service_spec *service);
+
+/** Returns if *service* is currently running.
+ *
+ * This function retuns true if the service has been started using
+ * *rte_service_start*, AND a service core is mapped to the service. This
+ * function can be used to ensure that the service will be run.
+ *
+ * @retval 1 Service is currently running, and has a service lcore mapped
+ * @retval 0 Service is currently stopped, or no service lcore is mapped
+ * @retval -EINVAL Invalid service pointer provided
+ */
+int32_t rte_service_is_running(const struct rte_service_spec *service);
+
+/** Start a service core.
+ *
+ * Starting a core makes the core begin polling. Any services assigned to it
+ * will be run as fast as possible.
+ *
+ * @retval 0 Success
+ * @retval -EINVAL Failed to start core. The *lcore_id* passed in is not
+ *          currently assigned to be a service core.
+ */
+int32_t rte_service_lcore_start(uint32_t lcore_id);
+
+/** Stop a service core.
+ *
+ * Stopping a core makes the core become idle, but remains  assigned as a
+ * service core.
+ *
+ * @retval 0 Success
+ * @retval -EINVAL Invalid *lcore_id* provided
+ * @retval -EALREADY Already stopped core
+ * @retval -EBUSY Failed to stop core, as it would cause a service to not
+ *          be run, as this is the only core currently running the service.
+ *          The application must stop the service first, and then stop the
+ *          lcore.
+ */
+int32_t rte_service_lcore_stop(uint32_t lcore_id);
+
+/** Adds lcore to the list of service cores.
+ *
+ * This functions can be used at runtime in order to modify the service core
+ * mask.
+ *
+ * @retval 0 Success
+ * @retval -EBUSY lcore is busy, and not available for service core duty
+ * @retval -EALREADY lcore is already added to the service core list
+ * @retval -EINVAL Invalid lcore provided
+ */
+int32_t rte_service_lcore_add(uint32_t lcore);
+
+/** Removes lcore from the list of service cores.
+ *
+ * This can fail if the core is not stopped, see *rte_service_core_stop*.
+ *
+ * @retval 0 Success
+ * @retval -EBUSY Lcore is not stopped, stop service core before removing.
+ * @retval -EINVAL failed to add lcore to service core mask.
+ */
+int32_t rte_service_lcore_del(uint32_t lcore);
+
+/** Retrieve the number of service cores currently available.
+ *
+ * This function returns the integer count of service cores available. The
+ * service core count can be used in mapping logic when creating mappings
+ * from service cores to services.
+ *
+ * See *rte_service_lcore_list* for details on retrieving the lcore_id of each
+ * service core.
+ *
+ * @return The number of service cores currently configured.
+ */
+int32_t rte_service_lcore_count(void);
+
+/** Reset all service core mappings.
+ * @retval 0 Success
+ */
+int32_t rte_service_lcore_reset_all(void);
+
+/** Enable or disable statistics collection.
+ *
+ * This function enables per core, per-service cycle count collection.
+ * @param enabled Zero to turn off statistics collection, non-zero to enable.
+ */
+void rte_service_set_stats_enable(int enabled);
+
+/** Retrieve the list of currently enabled service cores.
+ *
+ * This function fills in an application supplied array, with each element
+ * indicating the lcore_id of a service core.
+ *
+ * Adding and removing service cores can be performed using
+ * *rte_service_lcore_add* and *rte_service_lcore_del*.
+ * @param [out] array An array of at least N items.
+ * @param [out] The size of *array*.
+ * @retval >=0 Number of service cores that have been populated in the array
+ * @retval -ENOMEM The provided array is not large enough to fill in the
+ *          service core list. No items have been populated, call this function
+ *          with a size of at least *rte_service_core_count* items.
+ */
+int32_t rte_service_lcore_list(uint32_t array[], uint32_t n);
+
+/** Dumps any information available about the service. If service is NULL,
+ * dumps info for all services.
+ */
+int32_t rte_service_dump(FILE *f, struct rte_service_spec *service);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* _RTE_SERVICE_H_ */
diff --git a/lib/librte_eal/common/include/rte_service_private.h b/lib/librte_eal/common/include/rte_service_private.h
new file mode 100644
index 0000000..d518b02
--- /dev/null
+++ b/lib/librte_eal/common/include/rte_service_private.h
@@ -0,0 +1,118 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_SERVICE_PRIVATE_H_
+#define _RTE_SERVICE_PRIVATE_H_
+
+/* This file specifies the internal service specification.
+ * Include this file if you are writing a component that requires CPU cycles to
+ * operate, and you wish to run the component using service cores
+ */
+
+#include <rte_service.h>
+
+/**
+ * Signature of callback function to run a service.
+ */
+typedef int32_t (*rte_service_func)(void *args);
+
+/**
+ * The specification of a service.
+ *
+ * This struct contains metadata about the service itself, the callback
+ * function to run one iteration of the service, a userdata pointer, flags etc.
+ */
+struct rte_service_spec {
+	/** The name of the service. This should be used by the application to
+	 * understand what purpose this service provides.
+	 */
+	char name[RTE_SERVICE_NAME_MAX];
+	/** The callback to invoke to run one iteration of the service. */
+	rte_service_func callback;
+	/** The userdata pointer provided to the service callback. */
+	void *callback_userdata;
+	/** Flags to indicate the capabilities of this service. See defines in
+	 * the public header file for values of RTE_SERVICE_CAP_*
+	 */
+	uint32_t capabilities;
+	/** NUMA socket ID that this service is affinitized to */
+	int socket_id;
+};
+
+/** Register a new service.
+ *
+ * A service represents a component that the requires CPU time periodically to
+ * achieve its purpose.
+ *
+ * For example the eventdev SW PMD requires CPU cycles to perform its
+ * scheduling. This can be achieved by registering it as a service, and the
+ * application can then assign CPU resources to it using
+ * *rte_service_set_coremask*.
+ *
+ * @param spec The specification of the service to register
+ * @retval 0 Successfully registered the service.
+ *         -EINVAL Attempted to register an invalid service (eg, no callback
+ *         set)
+ */
+int32_t rte_service_register(const struct rte_service_spec *spec);
+
+/** Unregister a service.
+ *
+ * The service being removed must be stopped before calling this function.
+ *
+ * @retval 0 The service was successfully unregistered.
+ * @retval -EBUSY The service is currently running, stop the service before
+ *          calling unregister. No action has been taken.
+ */
+int32_t rte_service_unregister(struct rte_service_spec *service);
+
+/** Private function to allow EAL to initialized default mappings.
+ *
+ * This function iterates all the services, and maps then to the available
+ * cores. Based on the capabilities of the services, they are set to run on the
+ * available cores in a round-robin manner.
+ *
+ * @retval 0 Success
+ */
+int32_t rte_service_set_default_mapping(void);
+
+/** Initialize the service library.
+ *
+ * In order to use the service library, it must be initialized. EAL initializes
+ * the library at startup.
+ *
+ * @retval 0 Success
+ * @retval -EALREADY Service library is already initialized
+ */
+int32_t rte_service_init(void);
+
+#endif /* _RTE_SERVICE_PRIVATE_H_ */
diff --git a/lib/librte_eal/common/rte_service.c b/lib/librte_eal/common/rte_service.c
new file mode 100644
index 0000000..67338db
--- /dev/null
+++ b/lib/librte_eal/common/rte_service.c
@@ -0,0 +1,671 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_service.h>
+#include "include/rte_service_private.h"
+
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_common.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_atomic.h>
+#include <rte_memory.h>
+#include <rte_malloc.h>
+
+#define RTE_SERVICE_NUM_MAX 64
+
+#define SERVICE_F_REGISTERED 0
+
+/* runstates for services and lcores, denoting if they are active or not */
+#define RUNSTATE_STOPPED 0
+#define RUNSTATE_RUNNING 1
+
+/* internal representation of a service */
+struct rte_service_spec_impl {
+	/* public part of the struct */
+	struct rte_service_spec spec;
+
+	/* atomic lock that when set indicates a service core is currently
+	 * running this service callback. When not set, a core may take the
+	 * lock and then run the service callback.
+	 */
+	rte_atomic32_t execute_lock;
+
+	/* API set/get-able variables */
+	int32_t runstate;
+	uint8_t internal_flags;
+
+	/* per service statistics */
+	uint32_t num_mapped_cores;
+	uint64_t calls;
+	uint64_t cycles_spent;
+} __rte_cache_aligned;
+
+/* the internal values of a service core */
+struct core_state {
+	/* map of services IDs are run on this core */
+	uint64_t service_mask;
+	uint8_t runstate; /* running or stopped */
+	uint8_t is_service_core; /* set if core is currently a service core */
+	uint8_t collect_statistics; /* if set, measure cycle counts */
+
+	/* extreme statistics */
+	uint64_t calls_per_service[RTE_SERVICE_NUM_MAX];
+} __rte_cache_aligned;
+
+static uint32_t rte_service_count;
+static struct rte_service_spec_impl *rte_services;
+static struct core_state *cores_state;
+static uint32_t rte_service_library_initialized;
+
+int32_t rte_service_init(void)
+{
+	if (rte_service_library_initialized) {
+		printf("service library init() called, init flag %d\n",
+			rte_service_library_initialized);
+		return -EALREADY;
+	}
+
+	rte_services = rte_calloc("rte_services", RTE_SERVICE_NUM_MAX,
+			sizeof(struct rte_service_spec_impl),
+			RTE_CACHE_LINE_SIZE);
+	if (!rte_services) {
+		printf("error allocating rte services array\n");
+		return -ENOMEM;
+	}
+
+	cores_state = rte_calloc("rte_service_core_states", RTE_MAX_LCORE,
+			sizeof(struct core_state), RTE_CACHE_LINE_SIZE);
+	if (!cores_state) {
+		printf("error allocating core states array\n");
+		return -ENOMEM;
+	}
+
+	int i;
+	int count = 0;
+	struct rte_config *cfg = rte_eal_get_configuration();
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		if (lcore_config[i].core_role == ROLE_SERVICE) {
+			if ((unsigned)i == cfg->master_lcore)
+				continue;
+			rte_service_lcore_add(i);
+			count++;
+		}
+	}
+
+	rte_service_library_initialized = 1;
+	return 0;
+}
+
+void rte_service_set_stats_enable(int enabled)
+{
+	uint32_t i;
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		cores_state[i].collect_statistics = enabled;
+}
+
+/* returns 1 if service is registered and has not been unregistered
+ * Returns 0 if service never registered, or has been unregistered
+ */
+static inline int
+service_valid(uint32_t id) {
+	return !!(rte_services[id].internal_flags &
+		 (1 << SERVICE_F_REGISTERED));
+}
+
+uint32_t
+rte_service_get_count(void)
+{
+	return rte_service_count;
+}
+
+struct rte_service_spec *
+rte_service_get_by_id(uint32_t id)
+{
+	struct rte_service_spec *service = NULL;
+	if (id < rte_service_count)
+		service = (struct rte_service_spec *)&rte_services[id];
+
+	return service;
+}
+
+struct rte_service_spec *rte_service_get_by_name(const char *name)
+{
+	struct rte_service_spec *service = NULL;
+	int i;
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (service_valid(i) &&
+				strcmp(name, rte_services[i].spec.name) == 0)
+			service = (struct rte_service_spec *)&rte_services[i];
+			break;
+	}
+
+	return service;
+}
+
+const char *
+rte_service_get_name(const struct rte_service_spec *service)
+{
+	return service->name;
+}
+
+int32_t
+rte_service_probe_capability(const struct rte_service_spec *service,
+			     uint32_t capability)
+{
+	return service->capabilities & capability;
+}
+
+int32_t
+rte_service_is_running(const struct rte_service_spec *spec)
+{
+	const struct rte_service_spec_impl *impl =
+		(const struct rte_service_spec_impl *)spec;
+	if (!impl)
+		return -EINVAL;
+
+	return (impl->runstate == RUNSTATE_RUNNING) &&
+		(impl->num_mapped_cores > 0);
+}
+
+int32_t
+rte_service_register(const struct rte_service_spec *spec)
+{
+	uint32_t i;
+	int32_t free_slot = -1;
+
+	if (spec->callback == NULL || strlen(spec->name) == 0)
+		return -EINVAL;
+
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (!service_valid(i)) {
+			free_slot = i;
+			break;
+		}
+	}
+
+	if ((free_slot < 0) || (i == RTE_SERVICE_NUM_MAX))
+		return -ENOSPC;
+
+	struct rte_service_spec_impl *s = &rte_services[free_slot];
+	s->spec = *spec;
+	s->internal_flags |= (1 << SERVICE_F_REGISTERED);
+
+	rte_smp_wmb();
+	rte_service_count++;
+
+	return 0;
+}
+
+int32_t
+rte_service_unregister(struct rte_service_spec *spec)
+{
+	struct rte_service_spec_impl *s = NULL;
+	struct rte_service_spec_impl *spec_impl =
+		(struct rte_service_spec_impl *)spec;
+
+	uint32_t i;
+	uint32_t service_id;
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (&rte_services[i] == spec_impl) {
+			s = spec_impl;
+			service_id = i;
+			break;
+		}
+	}
+
+	if (!s)
+		return -EINVAL;
+
+	rte_service_count--;
+	rte_smp_wmb();
+
+	s->internal_flags &= ~(1 << SERVICE_F_REGISTERED);
+
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		cores_state[i].service_mask &= ~(1 << service_id);
+
+	memset(&rte_services[service_id], 0,
+			sizeof(struct rte_service_spec_impl));
+
+	return 0;
+}
+
+int32_t
+rte_service_start(struct rte_service_spec *service)
+{
+	struct rte_service_spec_impl *s =
+		(struct rte_service_spec_impl *)service;
+	s->runstate = RUNSTATE_RUNNING;
+	rte_smp_wmb();
+	return 0;
+}
+
+int32_t
+rte_service_stop(struct rte_service_spec *service)
+{
+	struct rte_service_spec_impl *s =
+		(struct rte_service_spec_impl *)service;
+	s->runstate = RUNSTATE_STOPPED;
+	rte_smp_wmb();
+	return 0;
+}
+
+static int32_t
+rte_service_runner_func(void *arg)
+{
+	RTE_SET_USED(arg);
+	uint32_t i;
+	const int lcore = rte_lcore_id();
+	struct core_state *cs = &cores_state[lcore];
+
+	while (cores_state[lcore].runstate == RUNSTATE_RUNNING) {
+		const uint64_t service_mask = cs->service_mask;
+		for (i = 0; i < rte_service_count; i++) {
+			struct rte_service_spec_impl *s = &rte_services[i];
+			if (s->runstate != RUNSTATE_RUNNING ||
+					!(service_mask & (1 << i)))
+				continue;
+
+			/* check if this is the only core mapped, else use
+			 * atomic to serialize cores mapped to this service
+			 */
+			uint32_t *lock = (uint32_t *)&s->execute_lock;
+			if ((s->spec.capabilities & RTE_SERVICE_CAP_MT_SAFE) ||
+					(s->num_mapped_cores == 1 ||
+					rte_atomic32_cmpset(lock, 0, 1))) {
+				void *userdata = s->spec.callback_userdata;
+
+				if (cs->collect_statistics) {
+					uint64_t start = rte_rdtsc();
+					s->spec.callback(userdata);
+					uint64_t end = rte_rdtsc();
+					s->cycles_spent += end - start;
+					cs->calls_per_service[i]++;
+					s->calls++;
+				} else {
+					cs->calls_per_service[i]++;
+					s->spec.callback(userdata);
+					s->calls++;
+				}
+
+				rte_atomic32_clear(&s->execute_lock);
+			}
+		}
+	}
+
+	lcore_config[lcore].state = WAIT;
+
+	return 0;
+}
+
+int32_t
+rte_service_lcore_count(void)
+{
+	int32_t count = 0;
+	uint32_t i;
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		count += cores_state[i].is_service_core;
+	return count;
+}
+
+int32_t
+rte_service_lcore_list(uint32_t array[], uint32_t n)
+{
+	uint32_t count = rte_service_lcore_count();
+	if (count > n)
+		return -ENOMEM;
+
+	if (!array)
+		return -EINVAL;
+
+	uint32_t i;
+	uint32_t idx = 0;
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct core_state *cs = &cores_state[i];
+		if (cs->is_service_core) {
+			array[idx] = i;
+			idx++;
+		}
+	}
+
+	return count;
+}
+
+int32_t
+rte_service_set_default_mapping(void)
+{
+	/* create a default mapping from cores to services, then start the
+	 * services to make them transparent to unaware applications.
+	 */
+	uint32_t i;
+	int ret;
+	uint32_t count = rte_service_get_count();
+
+	int32_t lcore_iter = 0;
+	uint32_t ids[RTE_MAX_LCORE];
+	int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE);
+
+	for (i = 0; i < count; i++) {
+		struct rte_service_spec *s = rte_service_get_by_id(i);
+		if (!s)
+			return -EINVAL;
+
+		/* if no lcores available as services cores, don't setup map.
+		 * This means app logic must add cores, and setup mappings
+		 */
+		if (lcore_count > 0) {
+			/* do 1:1 core mapping here, with each service getting
+			 * assigned a single core by default. Adding multiple
+			 * services should multiplex to a single core, or 1:1
+			 * if services == cores
+			 */
+			ret = rte_service_enable_on_lcore(s, ids[lcore_iter]);
+			if (ret)
+				return -ENODEV;
+		}
+
+		lcore_iter++;
+		if (lcore_iter >= lcore_count)
+			lcore_iter = 0;
+
+		ret = rte_service_start(s);
+		if (ret)
+			return -ENOEXEC;
+	}
+
+	return 0;
+}
+
+static int32_t
+service_update(struct rte_service_spec *service, uint32_t lcore,
+		uint32_t *set, uint32_t *enabled)
+{
+	uint32_t i;
+	int32_t sid = -1;
+
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if ((struct rte_service_spec *)&rte_services[i] == service &&
+				service_valid(i)) {
+			sid = i;
+			break;
+		}
+	}
+
+	if (sid == -1 || lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	if (!cores_state[lcore].is_service_core)
+		return -EINVAL;
+
+	if (set) {
+		if (*set) {
+			cores_state[lcore].service_mask |=  (1 << sid);
+			rte_services[sid].num_mapped_cores++;
+		} else {
+			cores_state[lcore].service_mask &= ~(1 << sid);
+			rte_services[sid].num_mapped_cores--;
+		}
+	}
+
+	if (enabled)
+		*enabled = (cores_state[lcore].service_mask & (1 << sid));
+
+	rte_smp_wmb();
+
+	return 0;
+}
+
+int32_t rte_service_get_enabled_on_lcore(struct rte_service_spec *service,
+					uint32_t lcore)
+{
+	uint32_t enabled;
+	int ret = service_update(service, lcore, 0, &enabled);
+	if (ret == 0)
+		return enabled;
+	return -EINVAL;
+}
+
+int32_t
+rte_service_enable_on_lcore(struct rte_service_spec *service, uint32_t lcore)
+{
+	uint32_t on = 1;
+	return service_update(service, lcore, &on, 0);
+}
+
+int32_t
+rte_service_disable_on_lcore(struct rte_service_spec *service, uint32_t lcore)
+{
+	uint32_t off = 0;
+	return service_update(service, lcore, &off, 0);
+}
+
+int32_t rte_service_lcore_reset_all(void)
+{
+	/* loop over cores, reset all to mask 0 */
+	uint32_t i;
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		cores_state[i].service_mask = 0;
+		cores_state[i].is_service_core = 0;
+		cores_state[i].runstate = RUNSTATE_STOPPED;
+	}
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++)
+		rte_services[i].num_mapped_cores = 0;
+
+	rte_smp_wmb();
+
+	return 0;
+}
+
+int32_t
+rte_service_lcore_add(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+	if (cores_state[lcore].is_service_core)
+		return -EALREADY;
+
+	lcore_config[lcore].core_role = ROLE_SERVICE;
+
+	cores_state[lcore].is_service_core = 1;
+	cores_state[lcore].service_mask = 0;
+	cores_state[lcore].runstate = RUNSTATE_STOPPED;
+
+	return 0;
+}
+
+int32_t
+rte_service_lcore_del(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	struct core_state *cs = &cores_state[lcore];
+	if (!cs->is_service_core)
+		return -EINVAL;
+
+	if (cs->runstate != RUNSTATE_STOPPED)
+		return -EBUSY;
+
+	lcore_config[lcore].core_role = ROLE_RTE;
+	cores_state[lcore].is_service_core = 0;
+
+	return 0;
+}
+
+int32_t
+rte_service_lcore_start(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	struct core_state *cs = &cores_state[lcore];
+	if (!cs->is_service_core)
+		return -EINVAL;
+
+	if (cs->runstate == RUNSTATE_RUNNING)
+		return -EALREADY;
+
+	/* set core to run state first, and then launch otherwise it will
+	 * return immediately as runstate keeps it in the service poll loop
+	 */
+	cores_state[lcore].runstate = RUNSTATE_RUNNING;
+
+	int ret = rte_eal_remote_launch(rte_service_runner_func, 0, lcore);
+	/* returns -EBUSY if the core is already launched, 0 on success */
+	return ret;
+}
+
+int32_t
+rte_service_lcore_stop(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	if (cores_state[lcore].runstate == RUNSTATE_STOPPED)
+		return -EALREADY;
+
+	uint32_t i;
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		int32_t enabled = cores_state[i].service_mask & (1 << i);
+		int32_t service_running = rte_services[i].runstate !=
+						RUNSTATE_STOPPED;
+		int32_t only_core = rte_services[i].num_mapped_cores == 1;
+
+		/* if the core is mapped, and the service is running, and this
+		 * is the only core that is mapped, the service would cease to
+		 * run if this core stopped, so fail instead.
+		 */
+		if (enabled && service_running && only_core)
+			return -EBUSY;
+	}
+
+	cores_state[lcore].runstate = RUNSTATE_STOPPED;
+
+	return 0;
+}
+
+static void
+rte_service_dump_one(FILE *f, struct rte_service_spec_impl *s,
+		     uint64_t all_cycles, uint32_t reset)
+{
+	/* avoid divide by zero */
+	if (all_cycles == 0)
+		all_cycles = 1;
+
+	int calls = 1;
+	if (s->calls != 0)
+		calls = s->calls;
+
+	float cycles_pct = (((float)s->cycles_spent) / all_cycles) * 100.f;
+	fprintf(f,
+			"  %s : %0.1f %%\tcalls %"PRIu64"\tcycles %"
+			PRIu64"\tavg: %"PRIu64"\n",
+			s->spec.name, cycles_pct, s->calls, s->cycles_spent,
+			s->cycles_spent / calls);
+
+	if (reset) {
+		s->cycles_spent = 0;
+		s->calls = 0;
+	}
+}
+
+static void
+service_dump_calls_per_lcore(FILE *f, uint32_t lcore, uint32_t reset)
+{
+	uint32_t i;
+	struct core_state *cs = &cores_state[lcore];
+
+	fprintf(f, "%02d\t", lcore);
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (!service_valid(i))
+			continue;
+		fprintf(f, "%"PRIu64"\t", cs->calls_per_service[i]);
+		if (reset)
+			cs->calls_per_service[i] = 0;
+	}
+	fprintf(f, "\n");
+}
+
+int32_t rte_service_dump(FILE *f, struct rte_service_spec *service)
+{
+	uint32_t i;
+
+	uint64_t total_cycles = 0;
+	for (i = 0; i < rte_service_count; i++) {
+		if (!service_valid(i))
+			continue;
+		total_cycles += rte_services[i].cycles_spent;
+	}
+
+	int print_no_collect_warning = 0;
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		if (cores_state[i].collect_statistics == 0)
+			print_no_collect_warning = 1;
+	if (print_no_collect_warning)
+		fprintf(f, "Warning; cycle counts not collectd; refer to rte_service_set_stats_enable\n");
+
+	if (service) {
+		struct rte_service_spec_impl *s =
+			(struct rte_service_spec_impl *)service;
+		fprintf(f, "Service %s Summary\n", s->spec.name);
+		uint32_t reset = 0;
+		rte_service_dump_one(f, s, total_cycles, reset);
+		return 0;
+	}
+
+	fprintf(f, "Services Summary\n");
+	for (i = 0; i < rte_service_count; i++) {
+		uint32_t reset = 1;
+		rte_service_dump_one(f, &rte_services[i], total_cycles, reset);
+	}
+
+	fprintf(f, "Service Cores Summary\n");
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		if (lcore_config[i].core_role != ROLE_SERVICE)
+			continue;
+
+		uint32_t reset = 0;
+		service_dump_calls_per_lcore(f, i, reset);
+	}
+
+	return 0;
+}
diff --git a/lib/librte_eal/linuxapp/eal/Makefile b/lib/librte_eal/linuxapp/eal/Makefile
index 8651e27..e6ab6c3 100644
--- a/lib/librte_eal/linuxapp/eal/Makefile
+++ b/lib/librte_eal/linuxapp/eal/Makefile
@@ -99,6 +99,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_malloc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_elem.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_heap.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_keepalive.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_service.c
 
 # from arch dir
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_cpuflags.c
diff --git a/lib/librte_eal/linuxapp/eal/eal_thread.c b/lib/librte_eal/linuxapp/eal/eal_thread.c
index 9f88530..831ba07 100644
--- a/lib/librte_eal/linuxapp/eal/eal_thread.c
+++ b/lib/librte_eal/linuxapp/eal/eal_thread.c
@@ -184,7 +184,14 @@ eal_thread_loop(__attribute__((unused)) void *arg)
 		ret = lcore_config[lcore_id].f(fct_arg);
 		lcore_config[lcore_id].ret = ret;
 		rte_wmb();
-		lcore_config[lcore_id].state = FINISHED;
+
+		/* when a service core returns, it should go directly to WAIT
+		 * state, because the application will not lcore_wait() for it.
+		 */
+		if (lcore_config[lcore_id].core_role == ROLE_SERVICE)
+			lcore_config[lcore_id].state = WAIT;
+		else
+			lcore_config[lcore_id].state = FINISHED;
 	}
 
 	/* never reached */
diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
index 670bab3..830d224 100644
--- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
@@ -198,3 +198,32 @@ DPDK_17.05 {
 	vfio_get_group_no;
 
 } DPDK_17.02;
+
+DPDK_17.08 {
+	global:
+
+	rte_service_disable_on_lcore;
+	rte_service_dump;
+	rte_service_enable_on_lcore;
+	rte_service_get_by_id;
+	rte_service_get_by_name;
+	rte_service_get_count;
+	rte_service_get_enabled_on_lcore;
+	rte_service_is_running;
+	rte_service_lcore_add;
+	rte_service_lcore_count;
+	rte_service_lcore_del;
+	rte_service_lcore_list;
+	rte_service_lcore_reset_all;
+	rte_service_lcore_start;
+	rte_service_lcore_stop;
+	rte_service_probe_capability;
+	rte_service_register;
+	rte_service_reset;
+	rte_service_set_stats_enable;
+	rte_service_start;
+	rte_service_stop;
+	rte_service_unregister;
+
+
+} DPDK_17.05;
-- 
2.7.4

[PATCH v3 2/7] service cores: EAL init changes

[Harry van Haaren]

This commit shows the changes required in rte_eal_init()
to transparently launch the service threads. The threads
are launched into the service worker functions here because
after rte_eal_init() the application is not gauranteed to
call any other DPDK API.

As the registration of services happens at initialization
time, the services that require CPU time are already available
when we reach the end of rte_eal_init().

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

---

v2 comments:
- Include BSD implementation (Jerin)
- Move details of core-tracking into rte_service_lcore_add(Jerin)
- Given there are changes other to suggested, not using Ack
---
 lib/librte_eal/bsdapp/eal/eal.c   | 22 ++++++++++++++++++++++
 lib/librte_eal/linuxapp/eal/eal.c | 23 +++++++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c
index 05f0c1f..4f7dcb3 100644
--- a/lib/librte_eal/bsdapp/eal/eal.c
+++ b/lib/librte_eal/bsdapp/eal/eal.c
@@ -653,6 +653,17 @@ rte_eal_init(int argc, char **argv)
 	rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER);
 	rte_eal_mp_wait_lcore();
 
+	/* initialize services first so vdevs can register during bus_probe.
+	 * Ignore return value of already initialized, this means EAL parameter
+	 * -s was used to set a service-core mask.
+	 */
+	ret = rte_service_init();
+	if (ret) {
+		rte_eal_init_alert("rte_service_init() failed\n");
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	/* Probe all the buses and devices/drivers on them */
 	if (rte_bus_probe()) {
 		rte_eal_init_alert("Cannot probe devices\n");
@@ -660,6 +671,17 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 
+	/* initialize default services configuration */
+	uint32_t service_cores[RTE_MAX_LCORE];
+	int count = rte_service_lcore_list(service_cores, RTE_MAX_LCORE);
+	for (i = 0; i < count; i++)
+		rte_service_lcore_start(service_cores[i]);
+	ret = rte_service_set_default_mapping();
+	if (ret) {
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	rte_eal_mcfg_complete();
 
 	return fctret;
diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
index 7c78f2d..d63dd87 100644
--- a/lib/librte_eal/linuxapp/eal/eal.c
+++ b/lib/librte_eal/linuxapp/eal/eal.c
@@ -78,6 +78,7 @@
 #include <rte_version.h>
 #include <rte_atomic.h>
 #include <malloc_heap.h>
+#include <rte_service_private.h>
 
 #include "eal_private.h"
 #include "eal_thread.h"
@@ -932,6 +933,17 @@ rte_eal_init(int argc, char **argv)
 	rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER);
 	rte_eal_mp_wait_lcore();
 
+	/* initialize services first so vdevs can register during bus_probe.
+	 * Ignore return value of already initialized, this means EAL parameter
+	 * -s was used to set a service-core mask.
+	 */
+	ret = rte_service_init();
+	if (ret) {
+		rte_eal_init_alert("rte_service_init() failed\n");
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	/* Probe all the buses and devices/drivers on them */
 	if (rte_bus_probe()) {
 		rte_eal_init_alert("Cannot probe devices\n");
@@ -939,6 +951,17 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 
+	/* initialize default services configuration */
+	uint32_t service_cores[RTE_MAX_LCORE];
+	int count = rte_service_lcore_list(service_cores, RTE_MAX_LCORE);
+	for (i = 0; i < count; i++)
+		rte_service_lcore_start(service_cores[i]);
+	ret = rte_service_set_default_mapping();
+	if (ret) {
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	rte_eal_mcfg_complete();
 
 	return fctret;
-- 
2.7.4

[PATCH v3 3/7] service cores: coremask parsing

[Harry van Haaren]

Add logic for parsing a coremask from EAL, which allows
the application to be unaware of the cores being taken from
its coremask.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

---

v2:
- Remove printf() (Jerin)
- Remove commented code (Jerin)
- simplified core tracking, no requirement on
  #include rte_service in EAL parsing anymore.
---
 lib/librte_eal/common/eal_common_options.c | 77 ++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)

diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index f470195..cee200c 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -61,6 +61,7 @@ const char
 eal_short_options[] =
 	"b:" /* pci-blacklist */
 	"c:" /* coremask */
+	"s:" /* service coremask */
 	"d:" /* driver */
 	"h"  /* help */
 	"l:" /* corelist */
@@ -267,6 +268,73 @@ static int xdigit2val(unsigned char c)
 }
 
 static int
+eal_parse_service_coremask(const char *coremask)
+{
+	struct rte_config *cfg = rte_eal_get_configuration();
+	int i, j, idx = 0;
+	unsigned int count = 0;
+	char c;
+	int val;
+
+	if (coremask == NULL)
+		return -1;
+	/* Remove all blank characters ahead and after .
+	 * Remove 0x/0X if exists.
+	 */
+	while (isblank(*coremask))
+		coremask++;
+	if (coremask[0] == '0' && ((coremask[1] == 'x')
+		|| (coremask[1] == 'X')))
+		coremask += 2;
+	i = strlen(coremask);
+	while ((i > 0) && isblank(coremask[i - 1]))
+		i--;
+
+	if (i == 0)
+		return -1;
+
+	for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
+		c = coremask[i];
+		if (isxdigit(c) == 0) {
+			/* invalid characters */
+			return -1;
+		}
+		val = xdigit2val(c);
+		for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE;
+				j++, idx++) {
+			if ((1 << j) & val) {
+				/* handle master lcore already parsed */
+				uint32_t lcore = idx;
+				if (master_lcore_parsed &&
+						cfg->master_lcore == lcore)
+					continue;
+
+				if (!lcore_config[idx].detected) {
+					RTE_LOG(ERR, EAL,
+						"lcore %u unavailable\n", idx);
+					return -1;
+				}
+				lcore_config[idx].core_role = ROLE_SERVICE;
+				count++;
+			}
+		}
+	}
+
+	for (; i >= 0; i--)
+		if (coremask[i] != '0')
+			return -1;
+
+	for (; idx < RTE_MAX_LCORE; idx++)
+		lcore_config[idx].core_index = -1;
+
+	if (count == 0)
+		return -1;
+
+	cfg->service_lcore_count = count;
+	return 0;
+}
+
+static int
 eal_parse_coremask(const char *coremask)
 {
 	struct rte_config *cfg = rte_eal_get_configuration();
@@ -409,6 +477,8 @@ eal_parse_master_lcore(const char *arg)
 	if (cfg->master_lcore >= RTE_MAX_LCORE)
 		return -1;
 	master_lcore_parsed = 1;
+	/* ensure master core is not used as service core */
+	lcore_config[cfg->master_lcore].core_role = ROLE_RTE;
 	return 0;
 }
 
@@ -826,6 +896,13 @@ eal_parse_common_option(int opt, const char *optarg,
 		}
 		core_parsed = 1;
 		break;
+	/* service coremask */
+	case 's':
+		if (eal_parse_service_coremask(optarg) < 0) {
+			RTE_LOG(ERR, EAL, "invalid service coremask\n");
+			return -1;
+		}
+		break;
 	/* size of memory */
 	case 'm':
 		conf->memory = atoi(optarg);
-- 
2.7.4

[PATCH v3 4/7] service cores: add unit tests

[Harry van Haaren]

Add a bunch of unit tests, to ensure that the service
core functions are operating as expected.

As part of these tests a dummy service is registered which
allows identifying if a service callback has been invoked
by using the CPU tick counter. This allows identifying if
functions to start and stop service lcores are actually having
effect.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

---

v2 changes;
- Rename variable to slcore_id (Jerin)
- Rename function to unregister_all() (Jerin)
- Fix typos (Jerin)
- Add unit test for get_by_name()
- Add unit tests (all suggestions by Jerin)
-- get_name()
-- Verify probe_capability API
-- Verify MT_SAFE capability (see code for details)
-- Verify rte_service_dump() API
---
 test/test/Makefile             |   2 +
 test/test/test_service_cores.c | 496 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 498 insertions(+)
 create mode 100644 test/test/test_service_cores.c

diff --git a/test/test/Makefile b/test/test/Makefile
index ee240be..61e296b 100644
--- a/test/test/Makefile
+++ b/test/test/Makefile
@@ -151,6 +151,8 @@ SRCS-y += test_interrupts.c
 SRCS-y += test_version.c
 SRCS-y += test_func_reentrancy.c
 
+SRCS-y += test_service_cores.c
+
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline.c
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_num.c
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_etheraddr.c
diff --git a/test/test/test_service_cores.c b/test/test/test_service_cores.c
new file mode 100644
index 0000000..f2f3a93
--- /dev/null
+++ b/test/test/test_service_cores.c
@@ -0,0 +1,496 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <rte_common.h>
+#include <rte_hexdump.h>
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_memcpy.h>
+#include <rte_cycles.h>
+
+#include <rte_service.h>
+#include <rte_service_private.h>
+
+#include "test.h"
+
+/* used as the service core ID */
+static uint32_t slcore_id;
+/* used as timestamp to detect if a service core is running */
+static uint64_t service_tick;
+
+#define SERVICE_DELAY 1
+
+#define DUMMY_SERVICE_NAME "dummy_service"
+#define MT_SAFE_SERVICE_NAME "mt_safe_service"
+
+static int
+testsuite_setup(void)
+{
+	/* assuming lcore 1 is available for service-core testing */
+	slcore_id = 1;
+	return TEST_SUCCESS;
+}
+
+static void
+testsuite_teardown(void)
+{
+	/* release service cores? */
+}
+
+static int32_t dummy_cb(void *args)
+{
+	RTE_SET_USED(args);
+	service_tick++;
+	rte_delay_ms(SERVICE_DELAY);
+	return 0;
+}
+
+
+static int32_t dummy_mt_safe_cb(void *args)
+{
+	/* Atomic checks to ensure MT safe services allow > 1 thread to
+	 * concurrently run the callback. The concept is as follows;
+	 * 1) if lock is available, take the lock then delay
+	 * 2) if first lock is taken, and a thread arrives in the CB, we know
+	 *    that 2 threads are running the callback at the same time: MT safe
+	 */
+	uint32_t *test_params = args;
+	uint32_t *atomic_lock = &test_params[0];
+	uint32_t *pass_test = &test_params[1];
+	int lock_taken = rte_atomic32_cmpset(atomic_lock, 0, 1);
+	if (lock_taken) {
+		/* delay with the lock held */
+		rte_delay_ms(250);
+		rte_atomic32_clear((rte_atomic32_t *)atomic_lock);
+	} else {
+		/* 2nd thread will fail to take lock, so set pass flag */
+		*pass_test = 1;
+	}
+
+	return 0;
+}
+
+/* unregister all services */
+static int
+unregister_all(void)
+{
+	uint32_t i;
+	struct rte_service_spec *dead = (struct rte_service_spec *)0xdead;
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_unregister(0),
+			"Unregistered NULL pointer");
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_unregister(dead),
+			"Unregistered invalid pointer");
+
+	uint32_t c = rte_service_get_count();
+	for (i = 0; i < c; i++) {
+		struct rte_service_spec *s = rte_service_get_by_id(i);
+		TEST_ASSERT_EQUAL(0, rte_service_unregister(s),
+				"Error unregistering a valid service");
+	}
+
+	rte_service_lcore_reset_all();
+
+	return TEST_SUCCESS;
+}
+
+/* register a single dummy service */
+static int
+dummy_register(void)
+{
+	/* make sure there are no remains from previous tests */
+	unregister_all();
+
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid callback");
+	service.callback = dummy_cb;
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid name");
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Failed to register valid service");
+
+	return TEST_SUCCESS;
+}
+
+/* verify get_by_name() service lookup */
+static int
+service_get_by_name(void)
+{
+	unregister_all();
+
+	/* ensure with no services registered returns NULL */
+	TEST_ASSERT_EQUAL(0, rte_service_get_by_name(DUMMY_SERVICE_NAME),
+			"Service get by name should return NULL");
+
+	/* register service */
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid callback");
+	service.callback = dummy_cb;
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid name");
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Failed to register valid service");
+
+	/* ensure with dummy services registered returns same ptr as ID */
+	struct rte_service_spec *s_by_id = rte_service_get_by_id(0);
+	TEST_ASSERT_EQUAL(s_by_id, rte_service_get_by_name(DUMMY_SERVICE_NAME),
+			"Service get_by_name should equal get_by_id()");
+
+	unregister_all();
+
+	/* ensure after unregister, get_by_name returns NULL */
+	TEST_ASSERT_EQUAL(0, rte_service_get_by_name(DUMMY_SERVICE_NAME),
+			"get by name should return NULL after unregister");
+
+	return TEST_SUCCESS;
+}
+
+/* verify probe of capabilities */
+static int
+service_probe_capability(void)
+{
+	unregister_all();
+
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	service.callback = dummy_cb;
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+	service.capabilities |= RTE_SERVICE_CAP_MT_SAFE;
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Register of MT SAFE service failed");
+
+	/* verify flag is enabled */
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+	int32_t mt = rte_service_probe_capability(s, RTE_SERVICE_CAP_MT_SAFE);
+	TEST_ASSERT_EQUAL(1, mt, "MT SAFE capability flag not set.");
+
+
+	unregister_all();
+
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	service.callback = dummy_cb;
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Register of non-MT safe service failed");
+
+	/* verify flag is enabled */
+	s = rte_service_get_by_id(0);
+	mt = rte_service_probe_capability(s, RTE_SERVICE_CAP_MT_SAFE);
+	TEST_ASSERT_EQUAL(0, mt, "MT SAFE cap flag set on non MT SAFE service");
+
+	return unregister_all();
+}
+
+/* verify the service name */
+static int
+service_name(void)
+{
+	struct rte_service_spec *service = rte_service_get_by_id(0);
+
+	int equal = strcmp(service->name, DUMMY_SERVICE_NAME);
+	TEST_ASSERT_EQUAL(0, equal, "Error: Service name not correct");
+
+	return unregister_all();
+}
+
+/* verify service dump */
+static int
+service_dump(void)
+{
+	struct rte_service_spec *service = rte_service_get_by_id(0);
+	rte_service_set_stats_enable(1);
+	rte_service_dump(stdout, service);
+	return unregister_all();
+}
+
+/* start and stop a service */
+static int
+service_start_stop(void)
+{
+	struct rte_service_spec *service = rte_service_get_by_id(0);
+
+	/* is_running() returns if service is running and slcore is mapped */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(1),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(service, 1),
+			"Enabling service core");
+
+	TEST_ASSERT_EQUAL(0, rte_service_is_running(service),
+			"Error: Service should be stopped");
+
+	TEST_ASSERT_EQUAL(0, rte_service_stop(service),
+			"Error: Service stopped returned non-zero");
+
+	TEST_ASSERT_EQUAL(0, rte_service_is_running(service),
+			"Error: Service is running - should be stopped");
+
+	TEST_ASSERT_EQUAL(0, rte_service_start(service),
+			"Error: Service start returned non-zero");
+
+	TEST_ASSERT_EQUAL(1, rte_service_is_running(service),
+			"Error: Service is not running");
+
+	return unregister_all();
+}
+
+/* enable and disable a lcore for a service */
+static int
+service_lcore_en_dis_able(void)
+{
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+
+	/* expected failure cases */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_enable_on_lcore(s, 100000),
+			"Enable on invalid core did not fail");
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_disable_on_lcore(s, 100000),
+			"Disable on invalid core did not fail");
+
+	/* add service core to allow enabling */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Add service core failed when not in use before");
+
+	/* valid enable */
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, slcore_id),
+			"Enabling valid service and core failed");
+	TEST_ASSERT_EQUAL(1, rte_service_get_enabled_on_lcore(s, slcore_id),
+			"Enabled core returned not-enabled");
+
+	/* valid disable */
+	TEST_ASSERT_EQUAL(0, rte_service_disable_on_lcore(s, slcore_id),
+			"Disabling valid service and lcore failed");
+	TEST_ASSERT_EQUAL(0, rte_service_get_enabled_on_lcore(s, slcore_id),
+			"Disabled core returned enabled");
+
+	return unregister_all();
+}
+
+static int
+service_lcore_running_check(void)
+{
+	uint64_t tick = service_tick;
+	rte_delay_ms(SERVICE_DELAY * 10);
+	/* if (tick != service_tick) we know the lcore as polled the service */
+	return tick != service_tick;
+}
+
+static int
+service_lcore_add_del(void)
+{
+	/* check initial count */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_count(),
+			"Service lcore count has value before adding a lcore");
+
+	/* check service lcore add */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Add service core failed when not in use before");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_add(slcore_id),
+			"Add service core failed to refuse in-use lcore");
+
+	/* check count */
+	TEST_ASSERT_EQUAL(1, rte_service_lcore_count(),
+			"Service core count not equal to one");
+
+	/* retrieve core list, checking lcore ids */
+	const uint32_t size = 4;
+	uint32_t service_core_ids[size];
+	int32_t n = rte_service_lcore_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(1, n, "Service core list return should equal 1");
+	TEST_ASSERT_EQUAL(slcore_id, service_core_ids[0],
+				"Service core list lcore must equal slcore_id");
+
+	/* recheck count, add more cores, and check count */
+	TEST_ASSERT_EQUAL(1, rte_service_lcore_count(),
+			"Service core count not equal to one");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(2),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(3),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(3, rte_service_lcore_count(),
+			"Service core count not equal to three");
+
+	/* check longer service core list */
+	n = rte_service_lcore_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(3, n, "Service core list return should equal 3");
+	TEST_ASSERT_EQUAL(slcore_id, service_core_ids[0],
+				"Service core list[0] lcore must equal 1");
+	TEST_ASSERT_EQUAL(2, service_core_ids[1],
+				"Service core list[1] lcore must equal 2");
+	TEST_ASSERT_EQUAL(3, service_core_ids[2],
+				"Service core list[2] lcore must equal 3");
+
+	/* recheck count, remove lcores, check remaining lcore_id is correct */
+	TEST_ASSERT_EQUAL(3, rte_service_lcore_count(),
+			"Service core count not equal to three");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_del(1),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_del(2),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(1, rte_service_lcore_count(),
+			"Service core count not equal to one");
+	n = rte_service_lcore_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(1, n, "Service core list return should equal one");
+	TEST_ASSERT_EQUAL(3, service_core_ids[0],
+				"Service core list[0] lcore must equal three");
+
+	return unregister_all();
+}
+
+static int
+service_mt_safe_poll(void)
+{
+	unregister_all();
+
+	/* add 2 cores */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(1), "mt safe lcore add fail");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(2), "mt safe lcore add fail");
+
+	/* Use atomic locks to verify that two threads are in the same function
+	 * at the same time. These are passed to the unit tests through the
+	 * callback userdata parameter
+	 */
+	uint32_t test_params[2];
+	memset(test_params, 0, sizeof(uint32_t) * 2);
+
+	/* register MT safe service. */
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	service.callback = dummy_mt_safe_cb;
+	service.callback_userdata = test_params;
+	snprintf(service.name, sizeof(service.name), MT_SAFE_SERVICE_NAME);
+	service.capabilities |= RTE_SERVICE_CAP_MT_SAFE;
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Register of MT SAFE service failed");
+
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+	TEST_ASSERT_EQUAL(0, rte_service_start(s),
+			"Starting valid service failed");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, 1),
+			"Failed to enable lcore 1 on mt safe service");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, 2),
+			"Failed to enable lcore 2 on mt safe service");
+	rte_service_lcore_start(1);
+	rte_service_lcore_start(2);
+
+	/* wait for the worker threads to run */
+	rte_delay_ms(500);
+	rte_service_lcore_stop(1);
+	rte_service_lcore_stop(2);
+
+	TEST_ASSERT_EQUAL(1, test_params[1],
+			"MT Safe service not run by two cores concurrently");
+
+	TEST_ASSERT_EQUAL(0, rte_service_stop(s),
+			"Failed to stop MT Safe service");
+
+	return unregister_all();
+}
+
+/* start and stop a service core - ensuring it goes back to sleep */
+static int
+service_lcore_start_stop(void)
+{
+	/* start service core and service, create mapping so tick() runs */
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+	TEST_ASSERT_EQUAL(0, rte_service_start(s),
+			"Starting valid service failed");
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_enable_on_lcore(s, slcore_id),
+			"Enabling valid service on non-service core must fail");
+
+	/* core start */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_lcore_start(slcore_id),
+			"Service core start without add should return EINVAL");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, slcore_id),
+			"Enabling valid service on valid core failed");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_start(slcore_id),
+			"Service core start after add failed");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_start(slcore_id),
+			"Service core expected as running but was stopped");
+
+	/* ensures core really is running the service function */
+	TEST_ASSERT_EQUAL(1, service_lcore_running_check(),
+			"Service core expected to poll service but it didn't");
+
+	/* core stop */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_lcore_stop(100000),
+			"Invalid Service core stop should return -EINVAL");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_stop(slcore_id),
+			"Service core stop expected to return 0");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_stop(slcore_id),
+			"Already stopped service core should return -EALREADY");
+
+	/* ensure service is not longer running */
+	TEST_ASSERT_EQUAL(0, service_lcore_running_check(),
+			"Service core expected to poll service but it didn't");
+
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_del(slcore_id),
+			"Service core del did not return zero");
+
+	return unregister_all();
+}
+
+static struct unit_test_suite service_tests  = {
+	.suite_name = "service core test suite",
+	.setup = testsuite_setup,
+	.teardown = testsuite_teardown,
+	.unit_test_cases = {
+		TEST_CASE_ST(dummy_register, NULL, unregister_all),
+		TEST_CASE_ST(dummy_register, NULL, service_name),
+		TEST_CASE_ST(dummy_register, NULL, service_get_by_name),
+		TEST_CASE_ST(dummy_register, NULL, service_dump),
+		TEST_CASE_ST(dummy_register, NULL, service_probe_capability),
+		TEST_CASE_ST(dummy_register, NULL, service_start_stop),
+		TEST_CASE_ST(dummy_register, NULL, service_lcore_add_del),
+		TEST_CASE_ST(dummy_register, NULL, service_lcore_start_stop),
+		TEST_CASE_ST(dummy_register, NULL, service_lcore_en_dis_able),
+		TEST_CASE_ST(dummy_register, NULL, service_mt_safe_poll),
+		TEST_CASES_END() /**< NULL terminate unit test array */
+	}
+};
+
+static int
+test_service_common(void)
+{
+	return unit_test_suite_runner(&service_tests);
+}
+
+REGISTER_TEST_COMMAND(service_autotest, test_service_common);
-- 
2.7.4

[PATCH v3 5/7] service cores: enable event/sw with service

[Harry van Haaren]

This commit shows how easy it is to enable a specific
DPDK component with a service callback, in order to get
CPU cycles for it.

The beauty of this method is that the service is unaware
of how much CPU time it is getting - the application can
decide how to split and slice cores and map them to the
registered services.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

---

v2:
- Remove #include <rte_cycles.h> (Jerin)
- Remove development prints (Jerin)
- Track service name in PMD
- Print warning if service does not have an lcore mapped (Jerin)
---
 drivers/event/sw/sw_evdev.c | 32 ++++++++++++++++++++++++++++++++
 drivers/event/sw/sw_evdev.h |  3 +++
 2 files changed, 35 insertions(+)

diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
index fe2a61e..baab376 100644
--- a/drivers/event/sw/sw_evdev.c
+++ b/drivers/event/sw/sw_evdev.c
@@ -38,6 +38,7 @@
 #include <rte_kvargs.h>
 #include <rte_ring.h>
 #include <rte_errno.h>
+#include <rte_service_private.h>
 
 #include "sw_evdev.h"
 #include "iq_ring.h"
@@ -597,6 +598,13 @@ sw_start(struct rte_eventdev *dev)
 {
 	unsigned int i, j;
 	struct sw_evdev *sw = sw_pmd_priv(dev);
+
+	/* check a service core is mapped to this service */
+	struct rte_service_spec *s = rte_service_get_by_name(sw->service_name);
+	if (!rte_service_is_running(s))
+		SW_LOG_ERR("Warning: No Service core enabled on service %s\n",
+				s->name);
+
 	/* check all ports are set up */
 	for (i = 0; i < sw->port_count; i++)
 		if (sw->ports[i].rx_worker_ring == NULL) {
@@ -699,6 +707,14 @@ set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque)
 	return 0;
 }
 
+
+static int32_t sw_sched_service_func(void *args)
+{
+	struct rte_eventdev *dev = args;
+	sw_event_schedule(dev);
+	return 0;
+}
+
 static int
 sw_probe(struct rte_vdev_device *vdev)
 {
@@ -810,6 +826,22 @@ sw_probe(struct rte_vdev_device *vdev)
 	sw->credit_update_quanta = credit_quanta;
 	sw->sched_quanta = sched_quanta;
 
+	/* register service with EAL */
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	snprintf(service.name, sizeof(service.name), "%s_service", name);
+	snprintf(sw->service_name, sizeof(sw->service_name), "%s_service",
+			name);
+	service.socket_id = socket_id;
+	service.callback = sw_sched_service_func;
+	service.callback_userdata = (void *)dev;
+
+	int32_t ret = rte_service_register(&service);
+	if (ret) {
+		SW_LOG_ERR("service register() failed");
+		return -ENOEXEC;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/event/sw/sw_evdev.h b/drivers/event/sw/sw_evdev.h
index 0d7f94f..3e83823 100644
--- a/drivers/event/sw/sw_evdev.h
+++ b/drivers/event/sw/sw_evdev.h
@@ -59,6 +59,7 @@
 
 #define EVENTDEV_NAME_SW_PMD event_sw
 #define SW_PMD_NAME RTE_STR(event_sw)
+#define SW_PMD_NAME_MAX 64
 
 #define SW_SCHED_TYPE_DIRECT (RTE_SCHED_TYPE_PARALLEL + 1)
 
@@ -276,6 +277,8 @@ struct sw_evdev {
 	/* store num stats and offset of the stats for each queue */
 	uint16_t xstats_count_per_qid[RTE_EVENT_MAX_QUEUES_PER_DEV];
 	uint16_t xstats_offset_for_qid[RTE_EVENT_MAX_QUEUES_PER_DEV];
+
+	char service_name[SW_PMD_NAME_MAX];
 };
 
 static inline struct sw_evdev *
-- 
2.7.4

[PATCH v3 6/7] maintainers: claim service cores

[Harry van Haaren]

Sign-up to be the maintainer of public header files and
implementation of the service-cores infrastructure.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
---
 MAINTAINERS | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/MAINTAINERS b/MAINTAINERS
index 00351ff..2e5081c 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -134,6 +134,12 @@ F: test/test/test_mp_secondary.c
 F: examples/multi_process/
 F: doc/guides/sample_app_ug/multi_process.rst
 
+Service Cores
+M: Harry van Haaren <harry.van.haaren@intel.com>
+F: lib/librte_eal/common/include/rte_service.h
+F: lib/librte_eal/common/include/rte_service_private.h
+F: lib/librte_eal/common/rte_keepalive.c
+
 ARM v7
 M: Jan Viktorin <viktorin@rehivetech.com>
 M: Jianbo Liu <jianbo.liu@linaro.org>
-- 
2.7.4

[PATCH v3 7/7] doc: add service cores to doc and release notes

[Harry van Haaren]

Add a section describing the fundamental concepts behind service
cores. Where service cores originate from, and how to enable services.
The release notes for 17.08 are updated, with an introductory paragraph
on the service cores concept. Finally the Eventdev SW PMD documentation
is amended to reflect that it can be run as a service.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

---

I would like to enable the service-cores in the eventdev_pipeline
sample app, to showcase the power of the service-core abstraction.
There is some remaining work TODO, in order to genericise the sample
app for both HW and SW PMDs, and during that rework the service-cores
can be added too.

The sample app will make a good showcase for docs, and make it much
easier to understand.
---
 doc/guides/eventdevs/sw.rst             |  4 +-
 doc/guides/prog_guide/index.rst         |  1 +
 doc/guides/prog_guide/service_cores.rst | 81 +++++++++++++++++++++++++++++++++
 doc/guides/rel_notes/release_17_08.rst  |  8 ++++
 4 files changed, 93 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/prog_guide/service_cores.rst

diff --git a/doc/guides/eventdevs/sw.rst b/doc/guides/eventdevs/sw.rst
index fb63c84..a3e6624 100644
--- a/doc/guides/eventdevs/sw.rst
+++ b/doc/guides/eventdevs/sw.rst
@@ -32,7 +32,9 @@ Software Eventdev Poll Mode Driver
 
 The software eventdev is an implementation of the eventdev API, that provides a
 wide range of the eventdev features. The eventdev relies on a CPU core to
-perform event scheduling.
+perform event scheduling. This PMD can use the service core library to run the
+scheduling function, allowing an application to utilize the power of service
+cores to multiplex other work on the same core if required.
 
 
 Features
diff --git a/doc/guides/prog_guide/index.rst b/doc/guides/prog_guide/index.rst
index ef5a02a..231622a 100644
--- a/doc/guides/prog_guide/index.rst
+++ b/doc/guides/prog_guide/index.rst
@@ -38,6 +38,7 @@ Programmer's Guide
     intro
     overview
     env_abstraction_layer
+    service_cores
     ring_lib
     mempool_lib
     mbuf_lib
diff --git a/doc/guides/prog_guide/service_cores.rst b/doc/guides/prog_guide/service_cores.rst
new file mode 100644
index 0000000..3a029ba
--- /dev/null
+++ b/doc/guides/prog_guide/service_cores.rst
@@ -0,0 +1,81 @@
+..  BSD LICENSE
+    Copyright(c) 2017 Intel Corporation. All rights reserved.
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions
+    are met:
+
+    * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+    * Neither the name of Intel Corporation nor the names of its
+    contributors may be used to endorse or promote products derived
+    from this software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Service Cores
+=============
+
+DPDK has a concept known as service cores, which enables a dynamic way of
+performing work on DPDK lcores. Service core support is built into the EAL, and
+an API is provided to optionally allow applications to control how the service
+cores are used at runtime.
+
+The service cores concept is built up out of services (components of DPDK that
+require CPU cycles to operate) and service cores (DPDK lcores, tasked with
+running services). The power of the service core concept is that the mapping
+between service cores and services can be configured to abstract away the
+difference between platforms and environments.
+
+For example, the Eventdev has hardware and software PMDs. Of these the software
+PMD requires an lcore to perform the scheduling operations, while the hardware
+PMD does not. With service cores, the application would not directly notice
+that the scheduling is done in software.
+
+For detailed information about the service core API, please refer to the docs.
+
+Service Core Initialization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are two methods to having service cores in a DPDK application, either by
+using the service coremask, or by dynamically adding cores using the API.
+The simpler of the two is to pass the `-s` coremask argument to EAL, which will
+take any cores available in the main DPDK coremask, an if the bits are also set
+in the service coremask the cores become service-cores instead of DPDK
+application lcores.
+
+Enabling Services on Cores
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Each registered service can be individually mapped to a service core, or set of
+service cores. Enabling a service on a particular core means that the lcore in
+question will run the service. Disabling that core on the service stops the
+lcore in question from running the service.
+
+Using this method, it is possible to assign specific workloads to each
+service core, and map N workloads to M number of service cores. Each service
+lcore loops over the services that are enabled for that core, and invokes the
+function to run the service.
+
+Service Core Statistics
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The service core library is capable of collecting runtime statistics like number
+of calls to a specific service, and number of cycles used by the service. The
+cycle count collection is dynamically configurable, allowing any application to
+profile the services running on the system at any time.
diff --git a/doc/guides/rel_notes/release_17_08.rst b/doc/guides/rel_notes/release_17_08.rst
index 842f46f..2c8e78a 100644
--- a/doc/guides/rel_notes/release_17_08.rst
+++ b/doc/guides/rel_notes/release_17_08.rst
@@ -41,6 +41,14 @@ New Features
      Also, make sure to start the actual text at the margin.
      =========================================================
 
+* **Added Service Core functionality.**
+
+  The service core functionality added to EAL allows DPDK to run services such
+  as SW PMDs on lcores without the application manually running them. The
+  service core infrastructure allows flexibility of running multiple services
+  on the same service lcore, and provides the application with powerful APIs to
+  configure the mapping from service lcores to services.
+
 * **Added Generic Flow API support to enic.**
 
   Flow API support for outer Ethernet, VLAN, IPv4, IPv6, UDP, TCP, SCTP, VxLAN
-- 
2.7.4

Re: [PATCH v3 7/7] doc: add service cores to doc and release notes

[Mcnamara, John]

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Harry van Haaren
> Sent: Sunday, July 2, 2017 10:35 PM
> To: dev@dpdk.org
> Cc: jerin.jacob@caviumnetworks.com; thomas@monjalon.net; Wiles, Keith
> <keith.wiles@intel.com>; Richardson, Bruce <bruce.richardson@intel.com>;
> Van Haaren, Harry <harry.van.haaren@intel.com>
> Subject: [dpdk-dev] [PATCH v3 7/7] doc: add service cores to doc and
> release notes
> 
> Add a section describing the fundamental concepts behind service cores.
> Where service cores originate from, and how to enable services.
> The release notes for 17.08 are updated, with an introductory paragraph on
> the service cores concept. Finally the Eventdev SW PMD documentation is
> amended to reflect that it can be run as a service.
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

Acked-by: John McNamara <john.mcnamara@intel.com>

Re: [PATCH v3 5/7] service cores: enable event/sw with service

[Jerin Jacob]

-----Original Message-----
> Date: Sun, 2 Jul 2017 22:35:12 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: jerin.jacob@caviumnetworks.com, thomas@monjalon.net,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH v3 5/7] service cores: enable event/sw with service
> X-Mailer: git-send-email 2.7.4
> 
> This commit shows how easy it is to enable a specific
> DPDK component with a service callback, in order to get
> CPU cycles for it.
> 
> The beauty of this method is that the service is unaware
> of how much CPU time it is getting - the application can
> decide how to split and slice cores and map them to the
> registered services.
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> 
> ---
> 
> v2:
> - Remove #include <rte_cycles.h> (Jerin)
> - Remove development prints (Jerin)
> - Track service name in PMD
> - Print warning if service does not have an lcore mapped (Jerin)
> ---
>  drivers/event/sw/sw_evdev.c | 32 ++++++++++++++++++++++++++++++++
>  drivers/event/sw/sw_evdev.h |  3 +++
>  2 files changed, 35 insertions(+)
> 
> diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
> index fe2a61e..baab376 100644
> --- a/drivers/event/sw/sw_evdev.c
> +++ b/drivers/event/sw/sw_evdev.c
> @@ -38,6 +38,7 @@
>  #include <rte_kvargs.h>
>  #include <rte_ring.h>
>  #include <rte_errno.h>
> +#include <rte_service_private.h>
>  
>  #include "sw_evdev.h"
>  #include "iq_ring.h"
> @@ -597,6 +598,13 @@ sw_start(struct rte_eventdev *dev)
>  {
>  	unsigned int i, j;
>  	struct sw_evdev *sw = sw_pmd_priv(dev);
> +
> +	/* check a service core is mapped to this service */
> +	struct rte_service_spec *s = rte_service_get_by_name(sw->service_name);
> +	if (!rte_service_is_running(s))
> +		SW_LOG_ERR("Warning: No Service core enabled on service %s\n",
> +				s->name);

For now, we can print just Warning. Once we remove schedule api from
eventdev then we can make it as error. Right?

Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

Re: [PATCH v3 6/7] maintainers: claim service cores

[Jerin Jacob]

-----Original Message-----
> Date: Sun, 2 Jul 2017 22:35:13 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: jerin.jacob@caviumnetworks.com, thomas@monjalon.net,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH v3 6/7] maintainers: claim service cores
> X-Mailer: git-send-email 2.7.4
> 
> Sign-up to be the maintainer of public header files and
> implementation of the service-cores infrastructure.
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> ---
>  MAINTAINERS | 6 ++++++
>  1 file changed, 6 insertions(+)
> 
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 00351ff..2e5081c 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -134,6 +134,12 @@ F: test/test/test_mp_secondary.c
>  F: examples/multi_process/
>  F: doc/guides/sample_app_ug/multi_process.rst
>  
> +Service Cores
> +M: Harry van Haaren <harry.van.haaren@intel.com>
> +F: lib/librte_eal/common/include/rte_service.h
> +F: lib/librte_eal/common/include/rte_service_private.h
> +F: lib/librte_eal/common/rte_keepalive.c

Wrong filename(rte_keepalive.c).

With above fix:
Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

> +
>  ARM v7
>  M: Jan Viktorin <viktorin@rehivetech.com>
>  M: Jianbo Liu <jianbo.liu@linaro.org>
> -- 
> 2.7.4
> 

Re: [PATCH v3 7/7] doc: add service cores to doc and release notes

[Jerin Jacob]

-----Original Message-----
> Date: Sun, 2 Jul 2017 22:35:14 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: jerin.jacob@caviumnetworks.com, thomas@monjalon.net,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH v3 7/7] doc: add service cores to doc and release notes
> X-Mailer: git-send-email 2.7.4
> 
> Add a section describing the fundamental concepts behind service
> cores. Where service cores originate from, and how to enable services.
> The release notes for 17.08 are updated, with an introductory paragraph
> on the service cores concept. Finally the Eventdev SW PMD documentation
> is amended to reflect that it can be run as a service.
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

> 
> ---

Re: [PATCH v3 4/7] service cores: add unit tests

[Jerin Jacob]

-----Original Message-----
> Date: Sun, 2 Jul 2017 22:35:11 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: jerin.jacob@caviumnetworks.com, thomas@monjalon.net,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH v3 4/7] service cores: add unit tests
> X-Mailer: git-send-email 2.7.4
> 
> Add a bunch of unit tests, to ensure that the service
> core functions are operating as expected.
> 
> As part of these tests a dummy service is registered which
> allows identifying if a service callback has been invoked
> by using the CPU tick counter. This allows identifying if
> functions to start and stop service lcores are actually having
> effect.
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> 
> ---
> 
> v2 changes;
> - Rename variable to slcore_id (Jerin)
> - Rename function to unregister_all() (Jerin)
> - Fix typos (Jerin)
> - Add unit test for get_by_name()
> - Add unit tests (all suggestions by Jerin)
> -- get_name()
> -- Verify probe_capability API
> -- Verify MT_SAFE capability (see code for details)
> -- Verify rte_service_dump() API

I think now UT is addressing all the APIS.
A few comments below.

> ---
> +testsuite_setup(void)
> +{
> +	/* assuming lcore 1 is available for service-core testing */
> +	slcore_id = 1;

Rather than assuming, How about taking enabled lcore for testing using 
rte_get_next_lcore()?

dummy_mt_safe_cb() tests using the second lcore now. So I think, you can
the get the enabled lcore for MT_SAFE using rte_get_next_lcore() and exit
if two lcores not available.

> +	return TEST_SUCCESS;
> +}
> +
> +
> +static struct unit_test_suite service_tests  = {
> +	.suite_name = "service core test suite",
> +	.setup = testsuite_setup,
> +	.teardown = testsuite_teardown,
> +	.unit_test_cases = {
> +		TEST_CASE_ST(dummy_register, NULL, unregister_all),
> +		TEST_CASE_ST(dummy_register, NULL, service_name),
> +		TEST_CASE_ST(dummy_register, NULL, service_get_by_name),
> +		TEST_CASE_ST(dummy_register, NULL, service_dump),
> +		TEST_CASE_ST(dummy_register, NULL, service_probe_capability),
> +		TEST_CASE_ST(dummy_register, NULL, service_start_stop),
> +		TEST_CASE_ST(dummy_register, NULL, service_lcore_add_del),
> +		TEST_CASE_ST(dummy_register, NULL, service_lcore_start_stop),
> +		TEST_CASE_ST(dummy_register, NULL, service_lcore_en_dis_able),
> +		TEST_CASE_ST(dummy_register, NULL, service_mt_safe_poll),
> +		TEST_CASES_END() /**< NULL terminate unit test array */

Regarding "Moving service lcore to/from rte lcore back and forth" test
reply in
http://dpdk.org/dev/patchwork/patch/25655/

I was thinking to enable the test by, calling service_lcore_del() and
then call remote launch on that lcore to check it is properly in
ROLE_RTE state. How about adding a test for the same to make sure the
"Moving service lcore to/from rte lcore back and forth" is OK.

With above comments:

Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

> +	}
> +};
> +
> +static int
> +test_service_common(void)
> +{
> +	return unit_test_suite_runner(&service_tests);
> +}
> +
> +REGISTER_TEST_COMMAND(service_autotest, test_service_common);
> -- 
> 2.7.4
> 

Re: [PATCH v3 2/7] service cores: EAL init changes

[Jerin Jacob]

-----Original Message-----
> Date: Sun, 2 Jul 2017 22:35:09 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: jerin.jacob@caviumnetworks.com, thomas@monjalon.net,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH v3 2/7] service cores: EAL init changes
> X-Mailer: git-send-email 2.7.4
> 
> This commit shows the changes required in rte_eal_init()
> to transparently launch the service threads. The threads
> are launched into the service worker functions here because
> after rte_eal_init() the application is not gauranteed to
> call any other DPDK API.
> 
> As the registration of services happens at initialization
> time, the services that require CPU time are already available
> when we reach the end of rte_eal_init().
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> 
> ---
> 
> v2 comments:
> - Include BSD implementation (Jerin)
> - Move details of core-tracking into rte_service_lcore_add(Jerin)
> - Given there are changes other to suggested, not using Ack
> ---
>  lib/librte_eal/bsdapp/eal/eal.c   | 22 ++++++++++++++++++++++
>  lib/librte_eal/linuxapp/eal/eal.c | 23 +++++++++++++++++++++++
>  2 files changed, 45 insertions(+)
> 
> diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c
> index 05f0c1f..4f7dcb3 100644
> --- a/lib/librte_eal/bsdapp/eal/eal.c
> +++ b/lib/librte_eal/bsdapp/eal/eal.c
> @@ -653,6 +653,17 @@ rte_eal_init(int argc, char **argv)
>  	rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER);
>  	rte_eal_mp_wait_lcore();
>  
> +	/* initialize services first so vdevs can register during bus_probe.
> +	 * Ignore return value of already initialized, this means EAL parameter
> +	 * -s was used to set a service-core mask.
> +	 */
> +	ret = rte_service_init();
> +	if (ret) {
> +		rte_eal_init_alert("rte_service_init() failed\n");
> +		rte_errno = ENOEXEC;
> +		return -1;
> +	}
> +
>  	/* Probe all the buses and devices/drivers on them */
>  	if (rte_bus_probe()) {
>  		rte_eal_init_alert("Cannot probe devices\n");
> @@ -660,6 +671,17 @@ rte_eal_init(int argc, char **argv)
>  		return -1;
>  	}
>  
> +	/* initialize default services configuration */
> +	uint32_t service_cores[RTE_MAX_LCORE];
> +	int count = rte_service_lcore_list(service_cores, RTE_MAX_LCORE);
> +	for (i = 0; i < count; i++)
> +		rte_service_lcore_start(service_cores[i]);
> +	ret = rte_service_set_default_mapping();
> +	if (ret) {
> +		rte_errno = ENOEXEC;
> +		return -1;
> +	}
> +
>  	rte_eal_mcfg_complete();
>  
>  	return fctret;
> diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
> index 7c78f2d..d63dd87 100644
> --- a/lib/librte_eal/linuxapp/eal/eal.c
> +++ b/lib/librte_eal/linuxapp/eal/eal.c
> @@ -78,6 +78,7 @@
>  #include <rte_version.h>
>  #include <rte_atomic.h>
>  #include <malloc_heap.h>
> +#include <rte_service_private.h>

Not included this header file for bsdapp compilation. It should fail to
compile. Right?

Re: [PATCH v3 3/7] service cores: coremask parsing

[Jerin Jacob]

-----Original Message-----
> Date: Sun, 2 Jul 2017 22:35:10 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: jerin.jacob@caviumnetworks.com, thomas@monjalon.net,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH v3 3/7] service cores: coremask parsing
> X-Mailer: git-send-email 2.7.4
> 
> Add logic for parsing a coremask from EAL, which allows
> the application to be unaware of the cores being taken from
> its coremask.
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> 
> diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
> index f470195..cee200c 100644
> --- a/lib/librte_eal/common/eal_common_options.c
> +++ b/lib/librte_eal/common/eal_common_options.c
> @@ -61,6 +61,7 @@ const char
>  eal_short_options[] =
>  	"b:" /* pci-blacklist */
>  	"c:" /* coremask */
> +	"s:" /* service coremask */
>  	"d:" /* driver */
>  	"h"  /* help */
>  	"l:" /* corelist */
> @@ -267,6 +268,73 @@ static int xdigit2val(unsigned char c)
>  }

Missing the --help update for service coremask details.

I think, EAL arguments are documented in another area of doc directory
as well. Update the documents.


>  
>  static int
> +eal_parse_service_coremask(const char *coremask)
> +{
> +	struct rte_config *cfg = rte_eal_get_configuration();
> +	int i, j, idx = 0;
> +	unsigned int count = 0;
> +	char c;
> +	int val;
> +
> +	if (coremask == NULL)
> +		return -1;
> +	/* Remove all blank characters ahead and after .
> +	 * Remove 0x/0X if exists.
> +	 */
> +	while (isblank(*coremask))
> +		coremask++;
> +	if (coremask[0] == '0' && ((coremask[1] == 'x')
> +		|| (coremask[1] == 'X')))
> +		coremask += 2;
> +	i = strlen(coremask);
> +	while ((i > 0) && isblank(coremask[i - 1]))
> +		i--;
> +
> +	if (i == 0)
> +		return -1;
> +
> +	for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
> +		c = coremask[i];
> +		if (isxdigit(c) == 0) {
> +			/* invalid characters */
> +			return -1;
> +		}
> +		val = xdigit2val(c);
> +		for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE;
> +				j++, idx++) {
> +			if ((1 << j) & val) {
> +				/* handle master lcore already parsed */
> +				uint32_t lcore = idx;
> +				if (master_lcore_parsed &&
> +						cfg->master_lcore == lcore)
> +					continue;
> +
> +				if (!lcore_config[idx].detected) {
> +					RTE_LOG(ERR, EAL,
> +						"lcore %u unavailable\n", idx);
> +					return -1;
> +				}
> +				lcore_config[idx].core_role = ROLE_SERVICE;

Why not to use rte_service_lcore_add(idx) here. So that in future some
changes we don't need to touch this file.

I added following code in unit testcase and I have 8 cores system. So I
was expecting cores prints from "0 3 4 5 6 7" as lcore 1 and 2 will be
stolen by service core. But it looks like RTE_LCORE_FOREACH not honoring
previous rte_service_lcore_add() functions.

testsuite_setup(void)
{
+       int i;
+       rte_service_lcore_add(1);
+       rte_service_lcore_add(2);
+
+       RTE_LCORE_FOREACH(i)
+               printf("cores %d\n", i);

Re: [PATCH v3 1/7] service cores: header and implementation

[Jerin Jacob]

-----Original Message-----
> Date: Sun, 2 Jul 2017 22:35:08 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: jerin.jacob@caviumnetworks.com, thomas@monjalon.net,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH v3 1/7] service cores: header and implementation
> X-Mailer: git-send-email 2.7.4
> 
> Add header files, update .map files with new service
> functions, and add the service header to the doxygen
> for building.
> 
> This service header API allows DPDK to use services as
> a concept of something that requires CPU cycles. An example
> is a PMD that runs in software to schedule events, where a
> hardware version exists that does not require a CPU.
> 
> The code presented here is based on an initial RFC:
> http://dpdk.org/ml/archives/dev/2017-May/065207.html
> This was then reworked, and RFC v2 with the changes posted:
> http://dpdk.org/ml/archives/dev/2017-June/067194.html
> 
> This is the fourth iteration of the service core concept,
> with 2 RFCs and this being v2 of the implementation.
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
> index a5bd108..2a93397 100644
> --- a/lib/librte_eal/common/Makefile
> +++ b/lib/librte_eal/common/Makefile
> @@ -41,6 +41,7 @@ INC += rte_eal_memconfig.h rte_malloc_heap.h
>  INC += rte_hexdump.h rte_devargs.h rte_bus.h rte_dev.h rte_vdev.h
>  INC += rte_pci_dev_feature_defs.h rte_pci_dev_features.h
>  INC += rte_malloc.h rte_keepalive.h rte_time.h
> +INC += rte_service.h rte_service_private.h

This will install rte_service_private.h file $RTE_TARGET. Based on
earlier email, You don't want to expose rte_service_private.h to
application. Right?
If so, I think, we remove rte_service_private.h from here and add CFLAGS
to point this header in local makefile of the _component_ which interested
in getting eal component specific functions.

>  
>  GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h
>  GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h
> diff --git a/lib/librte_eal/common/eal_common_lcore.c b/lib/librte_eal/common/eal_common_lcore.c
> + * called on it.
> + * @retval 0 Service successfully switched off
> + */
> +int32_t rte_service_stop(struct rte_service_spec *service);
> +
> +/** Returns if *service* is currently running.
> + *
> + * This function retuns true if the service has been started using

s/retuns/returns/

> +int32_t rte_service_lcore_reset_all(void);
> +
> +/** Enable or disable statistics collection.
> + *
> + * This function enables per core, per-service cycle count collection.
> + * @param enabled Zero to turn off statistics collection, non-zero to enable.
> + */
> +void rte_service_set_stats_enable(int enabled);

Since it it is a "set" function, better argument is "int enable"


> +
> +/** Retrieve the list of currently enabled service cores.
> + *
> + * This function fills in an application supplied array, with each element
> + * indicating the lcore_id of a service core.
> + *
> + * Adding and removing service cores can be performed using
> + * *rte_service_lcore_add* and *rte_service_lcore_del*.
> + * @param [out] array An array of at least N items.

What is N here. It should be RTE_MAX_LCORE. Right?

> + * @param [out] The size of *array*.
> + * @retval >=0 Number of service cores that have been populated in the array
> + * @retval -ENOMEM The provided array is not large enough to fill in the
> + *          service core list. No items have been populated, call this function
> + *          with a size of at least *rte_service_core_count* items.
> + */
> +int32_t rte_service_lcore_list(uint32_t array[], uint32_t n);
> +
> +	cores_state = rte_calloc("rte_service_core_states", RTE_MAX_LCORE,
> +			sizeof(struct core_state), RTE_CACHE_LINE_SIZE);
> +	if (!cores_state) {
> +		printf("error allocating core states array\n");
> +		return -ENOMEM;
> +	}
> +
> +	int i;
> +	int count = 0;
> +	struct rte_config *cfg = rte_eal_get_configuration();
> +	for (i = 0; i < RTE_MAX_LCORE; i++) {
> +		if (lcore_config[i].core_role == ROLE_SERVICE) {
> +			if ((unsigned)i == cfg->master_lcore)
> +				continue;
> +			rte_service_lcore_add(i);
> +			count++;
> +		}
> +	}
> +
> +	rte_service_library_initialized = 1;
> +	return 0;
> +}
> +
> +void rte_service_set_stats_enable(int enabled)

Since it it is a "set" function, better argument is "int enable"

> +{
> +	uint32_t i;
> +	for (i = 0; i < RTE_MAX_LCORE; i++)
> +		cores_state[i].collect_statistics = enabled;
> +}
> +
> +/* returns 1 if service is registered and has not been unregistered
> + * Returns 0 if service never registered, or has been unregistered
> + */
> +static inline int
> +service_valid(uint32_t id) {
> +	return !!(rte_services[id].internal_flags &
> +		 (1 << SERVICE_F_REGISTERED));
> +}
> +
> +uint32_t
> +rte_service_get_count(void)
> +{
> +	return rte_service_count;
> +}
> +
> +struct rte_service_spec *
> +rte_service_get_by_id(uint32_t id)
> +{
> +	struct rte_service_spec *service = NULL;
> +	if (id < rte_service_count)
> +		service = (struct rte_service_spec *)&rte_services[id];
> +
> +	return service;
> +}
> +
> +struct rte_service_spec *rte_service_get_by_name(const char *name)
> +{
> +	struct rte_service_spec *service = NULL;
> +	int i;
> +	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
> +		if (service_valid(i) &&
> +				strcmp(name, rte_services[i].spec.name) == 0)
> +			service = (struct rte_service_spec *)&rte_services[i];
> +			break;

"break" should be under "if" condition. ie { and } for the "if" condition is
missing


> +	}
> +
> +	return service;
> +}
> +
> +int32_t
> +rte_service_register(const struct rte_service_spec *spec)
> +{
> +	uint32_t i;
> +	int32_t free_slot = -1;
> +
> +	if (spec->callback == NULL || strlen(spec->name) == 0)
> +		return -EINVAL;
> +
> +	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
> +		if (!service_valid(i)) {
> +			free_slot = i;
> +			break;
> +		}
> +	}
> +
> +	if ((free_slot < 0) || (i == RTE_SERVICE_NUM_MAX))
> +		return -ENOSPC;
> +
> +	struct rte_service_spec_impl *s = &rte_services[free_slot];
> +	s->spec = *spec;
> +	s->internal_flags |= (1 << SERVICE_F_REGISTERED);
> +
> +	rte_smp_wmb();
> +	rte_service_count++;

My earlier comments on rte_smp_wmb() was in assumption that
rte_service_register() called be from worker cores.

Can rte_service_register() called from worker threads. No. Right?
if yes then we need an atomic variable to increment rte_service_count.


> +
> +	return 0;
> +}
> +
> +int32_t
> +rte_service_start(struct rte_service_spec *service)
> +{
> +	struct rte_service_spec_impl *s =
> +		(struct rte_service_spec_impl *)service;
> +	s->runstate = RUNSTATE_RUNNING;
> +	rte_smp_wmb();
> +	return 0;
> +}
> +
> +int32_t
> +rte_service_stop(struct rte_service_spec *service)
> +{
> +	struct rte_service_spec_impl *s =
> +		(struct rte_service_spec_impl *)service;
> +	s->runstate = RUNSTATE_STOPPED;
> +	rte_smp_wmb();
> +	return 0;
> +}
> +
> +static int32_t
> +rte_service_runner_func(void *arg)
> +{
> +	RTE_SET_USED(arg);
> +	uint32_t i;
> +	const int lcore = rte_lcore_id();
> +	struct core_state *cs = &cores_state[lcore];

Couple of comments in fastpath code. IMO, better to move "service_mask"
and access to global variable(so that it allocated from local stack) outside the loop.
global variable may share with other frequent write variables.


	const uint64_t service_mask = cs->service_mask;
	const uint32_t __rte_service_count = rte_service_count;

> +
> +	while (cores_state[lcore].runstate == RUNSTATE_RUNNING) {
> +		for (i = 0; i < rte_service_count; i++) {
> +			struct rte_service_spec_impl *s = &rte_services[i];
> +			if (s->runstate != RUNSTATE_RUNNING ||
> +					!(service_mask & (1 << i)))
> +				continue;
> +
> +			/* check if this is the only core mapped, else use
> +			 * atomic to serialize cores mapped to this service
> +			 */
> +			uint32_t *lock = (uint32_t *)&s->execute_lock;
> +			if ((s->spec.capabilities & RTE_SERVICE_CAP_MT_SAFE) ||
> +					(s->num_mapped_cores == 1 ||
> +					rte_atomic32_cmpset(lock, 0, 1))) {
> +				void *userdata = s->spec.callback_userdata;
> +
> +				if (cs->collect_statistics) {
> +					uint64_t start = rte_rdtsc();
> +					s->spec.callback(userdata);
> +					uint64_t end = rte_rdtsc();
> +					s->cycles_spent += end - start;
> +					cs->calls_per_service[i]++;
> +					s->calls++;
> +				} else {
> +					cs->calls_per_service[i]++;

Should we need this  in non stat configuration ?

> +					s->spec.callback(userdata);
> +					s->calls++;

Should we need this  in non stat configuration ?

> +				}
> +
> +				rte_atomic32_clear(&s->execute_lock);

Call this only when "rte_atomic32_cmpset" in action as it is costly for
normal case.

IMO, we need to have rte_smp_rmb() here to update the status of
cores_state[lcore].runstate or s->runstate(as it can be updated from other
lcores)

> +			}
> +		}
> +	}
> +
> +	lcore_config[lcore].state = WAIT;
> +
> +	return 0;
> +}
> +

Re: [PATCH v3 3/7] service cores: coremask parsing

[Van Haaren, Harry]

> From: Jerin Jacob [mailto:jerin.jacob@caviumnetworks.com]
> Sent: Tuesday, July 4, 2017 1:46 PM
> To: Van Haaren, Harry <harry.van.haaren@intel.com>
> Cc: dev@dpdk.org; thomas@monjalon.net; Wiles, Keith <keith.wiles@intel.com>; Richardson,
> Bruce <bruce.richardson@intel.com>
> Subject: Re: [PATCH v3 3/7] service cores: coremask parsing
> 
> -----Original Message-----
> > Date: Sun, 2 Jul 2017 22:35:10 +0100
> > From: Harry van Haaren <harry.van.haaren@intel.com>
> > To: dev@dpdk.org
> > CC: jerin.jacob@caviumnetworks.com, thomas@monjalon.net,
> >  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
> >  <harry.van.haaren@intel.com>
> > Subject: [PATCH v3 3/7] service cores: coremask parsing
> > X-Mailer: git-send-email 2.7.4
> >
> > Add logic for parsing a coremask from EAL, which allows
> > the application to be unaware of the cores being taken from
> > its coremask.
> >
> > Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> >
> > diff --git a/lib/librte_eal/common/eal_common_options.c
> b/lib/librte_eal/common/eal_common_options.c
> > index f470195..cee200c 100644
> > --- a/lib/librte_eal/common/eal_common_options.c
> > +++ b/lib/librte_eal/common/eal_common_options.c
> > @@ -61,6 +61,7 @@ const char
> >  eal_short_options[] =
> >  	"b:" /* pci-blacklist */
> >  	"c:" /* coremask */
> > +	"s:" /* service coremask */
> >  	"d:" /* driver */
> >  	"h"  /* help */
> >  	"l:" /* corelist */
> > @@ -267,6 +268,73 @@ static int xdigit2val(unsigned char c)
> >  }
> 
> Missing the --help update for service coremask details.
> 
> I think, EAL arguments are documented in another area of doc directory
> as well. Update the documents.

Will double check / fix this. Replying here now to advance discussion below; 

> >  static int
> > +eal_parse_service_coremask(const char *coremask)
> > +{
> > +	struct rte_config *cfg = rte_eal_get_configuration();
> > +	int i, j, idx = 0;
> > +	unsigned int count = 0;
> > +	char c;
> > +	int val;
> > +
> > +	if (coremask == NULL)
> > +		return -1;
> > +	/* Remove all blank characters ahead and after .
> > +	 * Remove 0x/0X if exists.
> > +	 */
> > +	while (isblank(*coremask))
> > +		coremask++;
> > +	if (coremask[0] == '0' && ((coremask[1] == 'x')
> > +		|| (coremask[1] == 'X')))
> > +		coremask += 2;
> > +	i = strlen(coremask);
> > +	while ((i > 0) && isblank(coremask[i - 1]))
> > +		i--;
> > +
> > +	if (i == 0)
> > +		return -1;
> > +
> > +	for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
> > +		c = coremask[i];
> > +		if (isxdigit(c) == 0) {
> > +			/* invalid characters */
> > +			return -1;
> > +		}
> > +		val = xdigit2val(c);
> > +		for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE;
> > +				j++, idx++) {
> > +			if ((1 << j) & val) {
> > +				/* handle master lcore already parsed */
> > +				uint32_t lcore = idx;
> > +				if (master_lcore_parsed &&
> > +						cfg->master_lcore == lcore)
> > +					continue;
> > +
> > +				if (!lcore_config[idx].detected) {
> > +					RTE_LOG(ERR, EAL,
> > +						"lcore %u unavailable\n", idx);
> > +					return -1;
> > +				}
> > +				lcore_config[idx].core_role = ROLE_SERVICE;
> 
> Why not to use rte_service_lcore_add(idx) here. So that in future some
> changes we don't need to touch this file.

The issue here is that the hugepages memory that service-cores requires is not available at this point. Keep in mind, the EAL parse-opts runs before almost anything else (makes sense, given we can specify e.g. --no-huge).

Given that there is not rte_malloc() available at this point, we have a few options:
1) Use existing allocated mem, e.g. the lcore_config[] array as above.
2) Delay the parsing of service-core mask until later. Breaks "parse -> validate-> config -> run" workflow.
3) Allocate temp memory to store the service-core indexes, and later free that back (feels hacky to me?)

Current scheme of (1) makes the most sense to me.


> I added following code in unit testcase and I have 8 cores system. So I
> was expecting cores prints from "0 3 4 5 6 7" as lcore 1 and 2 will be
> stolen by service core. But it looks like RTE_LCORE_FOREACH not honoring
> previous rte_service_lcore_add() functions.
> 
> testsuite_setup(void)
> {
> +       int i;
> +       rte_service_lcore_add(1);
> +       rte_service_lcore_add(2);
> +
> +       RTE_LCORE_FOREACH(i)
> +               printf("cores %d\n", i);


Root cause found - and fixed. If you don't strongly object to lcore_config[] method above, then I can prioritize this and try get a patchset up ASAP.

Re: [PATCH v3 3/7] service cores: coremask parsing

[Jerin Jacob]

-----Original Message-----
> Date: Thu, 6 Jul 2017 14:47:20 +0000
> From: "Van Haaren, Harry" <harry.van.haaren@intel.com>
> To: Jerin Jacob <jerin.jacob@caviumnetworks.com>
> CC: "dev@dpdk.org" <dev@dpdk.org>, "thomas@monjalon.net"
>  <thomas@monjalon.net>, "Wiles, Keith" <keith.wiles@intel.com>,
>  "Richardson, Bruce" <bruce.richardson@intel.com>
> Subject: RE: [PATCH v3 3/7] service cores: coremask parsing
> 
> > From: Jerin Jacob [mailto:jerin.jacob@caviumnetworks.com]
> > Sent: Tuesday, July 4, 2017 1:46 PM
> > To: Van Haaren, Harry <harry.van.haaren@intel.com>
> > Cc: dev@dpdk.org; thomas@monjalon.net; Wiles, Keith <keith.wiles@intel.com>; Richardson,
> > Bruce <bruce.richardson@intel.com>
> > Subject: Re: [PATCH v3 3/7] service cores: coremask parsing
> > 
> > -----Original Message-----
> > > Date: Sun, 2 Jul 2017 22:35:10 +0100
> > > From: Harry van Haaren <harry.van.haaren@intel.com>
> > > To: dev@dpdk.org
> > > CC: jerin.jacob@caviumnetworks.com, thomas@monjalon.net,
> > >  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
> > >  <harry.van.haaren@intel.com>
> > > Subject: [PATCH v3 3/7] service cores: coremask parsing
> > > X-Mailer: git-send-email 2.7.4
> > >
> > > Add logic for parsing a coremask from EAL, which allows
> > > the application to be unaware of the cores being taken from
> > > its coremask.
> > >
> > > Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> > >
> > > diff --git a/lib/librte_eal/common/eal_common_options.c
> > b/lib/librte_eal/common/eal_common_options.c
> > > index f470195..cee200c 100644
> > > --- a/lib/librte_eal/common/eal_common_options.c
> > > +++ b/lib/librte_eal/common/eal_common_options.c
> > > @@ -61,6 +61,7 @@ const char
> > >  eal_short_options[] =
> > >  	"b:" /* pci-blacklist */
> > >  	"c:" /* coremask */
> > > +	"s:" /* service coremask */
> > >  	"d:" /* driver */
> > >  	"h"  /* help */
> > >  	"l:" /* corelist */
> > > @@ -267,6 +268,73 @@ static int xdigit2val(unsigned char c)
> > >  }
> > 
> > Missing the --help update for service coremask details.
> > 
> > I think, EAL arguments are documented in another area of doc directory
> > as well. Update the documents.
> 
> Will double check / fix this. Replying here now to advance discussion below; 
> 
> > >  static int
> > > +eal_parse_service_coremask(const char *coremask)
> > > +{
> > > +	struct rte_config *cfg = rte_eal_get_configuration();
> > > +	int i, j, idx = 0;
> > > +	unsigned int count = 0;
> > > +	char c;
> > > +	int val;
> > > +
> > > +	if (coremask == NULL)
> > > +		return -1;
> > > +	/* Remove all blank characters ahead and after .
> > > +	 * Remove 0x/0X if exists.
> > > +	 */
> > > +	while (isblank(*coremask))
> > > +		coremask++;
> > > +	if (coremask[0] == '0' && ((coremask[1] == 'x')
> > > +		|| (coremask[1] == 'X')))
> > > +		coremask += 2;
> > > +	i = strlen(coremask);
> > > +	while ((i > 0) && isblank(coremask[i - 1]))
> > > +		i--;
> > > +
> > > +	if (i == 0)
> > > +		return -1;
> > > +
> > > +	for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
> > > +		c = coremask[i];
> > > +		if (isxdigit(c) == 0) {
> > > +			/* invalid characters */
> > > +			return -1;
> > > +		}
> > > +		val = xdigit2val(c);
> > > +		for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE;
> > > +				j++, idx++) {
> > > +			if ((1 << j) & val) {
> > > +				/* handle master lcore already parsed */
> > > +				uint32_t lcore = idx;
> > > +				if (master_lcore_parsed &&
> > > +						cfg->master_lcore == lcore)
> > > +					continue;
> > > +
> > > +				if (!lcore_config[idx].detected) {
> > > +					RTE_LOG(ERR, EAL,
> > > +						"lcore %u unavailable\n", idx);
> > > +					return -1;
> > > +				}
> > > +				lcore_config[idx].core_role = ROLE_SERVICE;
> > 
> > Why not to use rte_service_lcore_add(idx) here. So that in future some
> > changes we don't need to touch this file.
> 
> The issue here is that the hugepages memory that service-cores requires is not available at this point. Keep in mind, the EAL parse-opts runs before almost anything else (makes sense, given we can specify e.g. --no-huge).
> 
> Given that there is not rte_malloc() available at this point, we have a few options:
> 1) Use existing allocated mem, e.g. the lcore_config[] array as above.
> 2) Delay the parsing of service-core mask until later. Breaks "parse -> validate-> config -> run" workflow.
> 3) Allocate temp memory to store the service-core indexes, and later free that back (feels hacky to me?)
> 
> Current scheme of (1) makes the most sense to me.

Yes. Make sense to keep option 1.

One suggestion:
There is a lot duplicate code between new eal_parse_service_coremask() and
eal_parse_coremask() on the same file. I think, we can add a common parsing logic
and on match, the actions can be invoked through function pointer which
is passed in the parsing function.

> 
> 
> > I added following code in unit testcase and I have 8 cores system. So I
> > was expecting cores prints from "0 3 4 5 6 7" as lcore 1 and 2 will be
> > stolen by service core. But it looks like RTE_LCORE_FOREACH not honoring
> > previous rte_service_lcore_add() functions.
> > 
> > testsuite_setup(void)
> > {
> > +       int i;
> > +       rte_service_lcore_add(1);
> > +       rte_service_lcore_add(2);
> > +
> > +       RTE_LCORE_FOREACH(i)
> > +               printf("cores %d\n", i);
> 
> 
> Root cause found - and fixed. If you don't strongly object to lcore_config[] method above, then I can prioritize this and try get a patchset up ASAP.

Great!!

> 

Re: [PATCH v3 3/7] service cores: coremask parsing

[Van Haaren, Harry]

> From: Jerin Jacob [mailto:jerin.jacob@caviumnetworks.com]
> Sent: Friday, July 7, 2017 11:46 AM
> To: Van Haaren, Harry <harry.van.haaren@intel.com>
> Cc: dev@dpdk.org; thomas@monjalon.net; Wiles, Keith <keith.wiles@intel.com>; Richardson,
> Bruce <bruce.richardson@intel.com>
> Subject: Re: [PATCH v3 3/7] service cores: coremask parsing
> 
> -----Original Message-----
> > Date: Thu, 6 Jul 2017 14:47:20 +0000
> > From: "Van Haaren, Harry" <harry.van.haaren@intel.com>
> > To: Jerin Jacob <jerin.jacob@caviumnetworks.com>
> > CC: "dev@dpdk.org" <dev@dpdk.org>, "thomas@monjalon.net"
> >  <thomas@monjalon.net>, "Wiles, Keith" <keith.wiles@intel.com>,
> >  "Richardson, Bruce" <bruce.richardson@intel.com>
> > Subject: RE: [PATCH v3 3/7] service cores: coremask parsing
> >
> > > From: Jerin Jacob [mailto:jerin.jacob@caviumnetworks.com]
> > > Sent: Tuesday, July 4, 2017 1:46 PM
> > > To: Van Haaren, Harry <harry.van.haaren@intel.com>
> > > Cc: dev@dpdk.org; thomas@monjalon.net; Wiles, Keith <keith.wiles@intel.com>;
> Richardson,
> > > Bruce <bruce.richardson@intel.com>
> > > Subject: Re: [PATCH v3 3/7] service cores: coremask parsing
> > >
> > > -----Original Message-----
> > > > Date: Sun, 2 Jul 2017 22:35:10 +0100
> > > > From: Harry van Haaren <harry.van.haaren@intel.com>
> > > > To: dev@dpdk.org
> > > > CC: jerin.jacob@caviumnetworks.com, thomas@monjalon.net,
> > > >  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
> > > >  <harry.van.haaren@intel.com>
> > > > Subject: [PATCH v3 3/7] service cores: coremask parsing
> > > > X-Mailer: git-send-email 2.7.4
> > > >
> > > > Add logic for parsing a coremask from EAL, which allows
> > > > the application to be unaware of the cores being taken from
> > > > its coremask.
> > > >
> > > > Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> > > >
> > > > diff --git a/lib/librte_eal/common/eal_common_options.c
> > > b/lib/librte_eal/common/eal_common_options.c
> > > > index f470195..cee200c 100644
> > > > --- a/lib/librte_eal/common/eal_common_options.c
> > > > +++ b/lib/librte_eal/common/eal_common_options.c
> > > > @@ -61,6 +61,7 @@ const char
> > > >  eal_short_options[] =
> > > >  	"b:" /* pci-blacklist */
> > > >  	"c:" /* coremask */
> > > > +	"s:" /* service coremask */
> > > >  	"d:" /* driver */
> > > >  	"h"  /* help */
> > > >  	"l:" /* corelist */
> > > > @@ -267,6 +268,73 @@ static int xdigit2val(unsigned char c)
> > > >  }
> > >
> > > Missing the --help update for service coremask details.
> > >
> > > I think, EAL arguments are documented in another area of doc directory
> > > as well. Update the documents.
> >
> > Will double check / fix this. Replying here now to advance discussion below;
> >
> > > >  static int
> > > > +eal_parse_service_coremask(const char *coremask)
> > > > +{
> > > > +	struct rte_config *cfg = rte_eal_get_configuration();
> > > > +	int i, j, idx = 0;
> > > > +	unsigned int count = 0;
> > > > +	char c;
> > > > +	int val;
> > > > +
> > > > +	if (coremask == NULL)
> > > > +		return -1;
> > > > +	/* Remove all blank characters ahead and after .
> > > > +	 * Remove 0x/0X if exists.
> > > > +	 */
> > > > +	while (isblank(*coremask))
> > > > +		coremask++;
> > > > +	if (coremask[0] == '0' && ((coremask[1] == 'x')
> > > > +		|| (coremask[1] == 'X')))
> > > > +		coremask += 2;
> > > > +	i = strlen(coremask);
> > > > +	while ((i > 0) && isblank(coremask[i - 1]))
> > > > +		i--;
> > > > +
> > > > +	if (i == 0)
> > > > +		return -1;
> > > > +
> > > > +	for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
> > > > +		c = coremask[i];
> > > > +		if (isxdigit(c) == 0) {
> > > > +			/* invalid characters */
> > > > +			return -1;
> > > > +		}
> > > > +		val = xdigit2val(c);
> > > > +		for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE;
> > > > +				j++, idx++) {
> > > > +			if ((1 << j) & val) {
> > > > +				/* handle master lcore already parsed */
> > > > +				uint32_t lcore = idx;
> > > > +				if (master_lcore_parsed &&
> > > > +						cfg->master_lcore == lcore)
> > > > +					continue;
> > > > +
> > > > +				if (!lcore_config[idx].detected) {
> > > > +					RTE_LOG(ERR, EAL,
> > > > +						"lcore %u unavailable\n", idx);
> > > > +					return -1;
> > > > +				}
> > > > +				lcore_config[idx].core_role = ROLE_SERVICE;
> > >
> > > Why not to use rte_service_lcore_add(idx) here. So that in future some
> > > changes we don't need to touch this file.
> >
> > The issue here is that the hugepages memory that service-cores requires is not available
> at this point. Keep in mind, the EAL parse-opts runs before almost anything else (makes
> sense, given we can specify e.g. --no-huge).
> >
> > Given that there is not rte_malloc() available at this point, we have a few options:
> > 1) Use existing allocated mem, e.g. the lcore_config[] array as above.
> > 2) Delay the parsing of service-core mask until later. Breaks "parse -> validate->
> config -> run" workflow.
> > 3) Allocate temp memory to store the service-core indexes, and later free that back
> (feels hacky to me?)
> >
> > Current scheme of (1) makes the most sense to me.
> 
> Yes. Make sense to keep option 1.

Great,

> One suggestion:
> There is a lot duplicate code between new eal_parse_service_coremask() and
> eal_parse_coremask() on the same file. I think, we can add a common parsing logic
> and on match, the actions can be invoked through function pointer which
> is passed in the parsing function.

Agreed - lets leave this cleanup of EAL to another patchset. That patchset can consolidate the various coremask functions in EAL and other parts of DPDK (I recall a similar comment for eventdev_pipeline sample-app).


> > > I added following code in unit testcase and I have 8 cores system. So I
> > > was expecting cores prints from "0 3 4 5 6 7" as lcore 1 and 2 will be
> > > stolen by service core. But it looks like RTE_LCORE_FOREACH not honoring
> > > previous rte_service_lcore_add() functions.
> > >
> > > testsuite_setup(void)
> > > {
> > > +       int i;
> > > +       rte_service_lcore_add(1);
> > > +       rte_service_lcore_add(2);
> > > +
> > > +       RTE_LCORE_FOREACH(i)
> > > +               printf("cores %d\n", i);
> >
> >
> > Root cause found - and fixed. If you don't strongly object to lcore_config[] method
> above, then I can prioritize this and try get a patchset up ASAP.
> 
> Great!!

I'm working on improving one of the unit tests to launch app functions to verify ROLE_RTE, and will post then.

Thanks for the input, -Harry

Re: [PATCH v3 5/7] service cores: enable event/sw with service

[Van Haaren, Harry]

> From: Jerin Jacob [mailto:jerin.jacob@caviumnetworks.com]
> Sent: Tuesday, July 4, 2017 11:53 AM
> To: Van Haaren, Harry <harry.van.haaren@intel.com>
> Cc: dev@dpdk.org; thomas@monjalon.net; Wiles, Keith <keith.wiles@intel.com>; Richardson,
> Bruce <bruce.richardson@intel.com>
> Subject: Re: [PATCH v3 5/7] service cores: enable event/sw with service
> 
> -----Original Message-----
> > Date: Sun, 2 Jul 2017 22:35:12 +0100
> > From: Harry van Haaren <harry.van.haaren@intel.com>
> > To: dev@dpdk.org
> > CC: jerin.jacob@caviumnetworks.com, thomas@monjalon.net,
> >  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
> >  <harry.van.haaren@intel.com>
> > Subject: [PATCH v3 5/7] service cores: enable event/sw with service
> > X-Mailer: git-send-email 2.7.4
> >
> > This commit shows how easy it is to enable a specific
> > DPDK component with a service callback, in order to get
> > CPU cycles for it.
> >
> > The beauty of this method is that the service is unaware
> > of how much CPU time it is getting - the application can
> > decide how to split and slice cores and map them to the
> > registered services.
> >
> > Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

<snip>

> > +
> > +	/* check a service core is mapped to this service */
> > +	struct rte_service_spec *s = rte_service_get_by_name(sw->service_name);
> > +	if (!rte_service_is_running(s))
> > +		SW_LOG_ERR("Warning: No Service core enabled on service %s\n",
> > +				s->name);
> 
> For now, we can print just Warning. Once we remove schedule api from
> eventdev then we can make it as error. Right?

Yes correct.

> Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

Thanks, will include in v+1.

Re: [PATCH v3 2/7] service cores: EAL init changes

[Van Haaren, Harry]

> From: Jerin Jacob [mailto:jerin.jacob@caviumnetworks.com]
> Sent: Tuesday, July 4, 2017 12:36 PM
> To: Van Haaren, Harry <harry.van.haaren@intel.com>
> Cc: dev@dpdk.org; thomas@monjalon.net; Wiles, Keith <keith.wiles@intel.com>; Richardson,
> Bruce <bruce.richardson@intel.com>
> Subject: Re: [PATCH v3 2/7] service cores: EAL init changes
> 
> -----Original Message-----
> > Date: Sun, 2 Jul 2017 22:35:09 +0100
> > From: Harry van Haaren <harry.van.haaren@intel.com>
> > To: dev@dpdk.org
> > CC: jerin.jacob@caviumnetworks.com, thomas@monjalon.net,
> >  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
> >  <harry.van.haaren@intel.com>
> > Subject: [PATCH v3 2/7] service cores: EAL init changes
> > X-Mailer: git-send-email 2.7.4
> >
> > This commit shows the changes required in rte_eal_init()
> > to transparently launch the service threads. The threads
> > are launched into the service worker functions here because
> > after rte_eal_init() the application is not gauranteed to
> > call any other DPDK API.
> >
> > As the registration of services happens at initialization
> > time, the services that require CPU time are already available
> > when we reach the end of rte_eal_init().
> >
> > Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> >
> > ---

<snip>

> > diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
> > index 7c78f2d..d63dd87 100644
> > --- a/lib/librte_eal/linuxapp/eal/eal.c
> > +++ b/lib/librte_eal/linuxapp/eal/eal.c
> > @@ -78,6 +78,7 @@
> >  #include <rte_version.h>
> >  #include <rte_atomic.h>
> >  #include <malloc_heap.h>
> > +#include <rte_service_private.h>
> 
> Not included this header file for bsdapp compilation. It should fail to
> compile. Right?

Fixed.

[PATCH v4 0/7] service cores: cover letter

[Harry van Haaren]

This patchset introduces service cores to DPDK. A service core
is an lcore that performs functions to abstract away details of
differences in environment of the application.

An example is using the eventdev API, where either a software or hardware
PMD performs scheduling. In the case of the software PMD an lcore is
required to perform scheduling, which means application logic would have
to be aware of the PMD running under the API. To abstract away the
differences in HW / SW PMDs, service cores can run the SW PMD service
without application logic specifying the exact cores to use. Note that
eventdev is only one API that benefits; timers, interrupts handling,
statistics and monitoring, and a range of other infrastructure that
requires a slice of CPU time may all benefit from service cores.

The application is not obliged to manually use the service cores API,
however if an application wishes to use the service cores API for fine
grained control over how the services are run, this is possible. Deciding
between a performance threading-profile and scaled-down profile can be
achieved by advanced usage of service cores and setting the lcore mappings.

Patch 5/7 shows how a PMD can register a service to run a function. This
is then available (along with any other registered services) to be run by
the service cores. Patches 6/7 and 7/7 add documentation, and claim
maintainership.

Regards, -Harry


v4:
- Range of fixes as suggested by Jerin
- Improved unit tests, ensuring ex-service cores become available to app
- Added functions to EXPERIMENTAL tag in .map files (Thomas)
- Added @warning experimental notes to Doxygen API documentation (Thomas)
- Various smaller fixes / cleanups
- See commit notes for details

v3:
- Added docs
- Added release notes
- Updated maintainers file
- Compile checks with devtools/test-build.sh
- Validated patches apply to latest dpdk/master
- Based on discussion, rte_service_iterate() is *not* included,
  but could be adding at a later date if use-cases require it.
- Future work includes enabling the eventdev_pipeline sample app, but there
  is still some churn there to enable both HW/SW PMDs seamlessly. Once sample
  app is enabled a service core walk-through with that sample app can be added
  to the docs, to provide a tutorial on service-core usage.

Harry van Haaren (7):
  service cores: header and implementation
  service cores: EAL init changes
  service cores: coremask parsing
  service cores: add unit tests
  event/sw: enable SW PMD with service capability
  doc: add service cores to doc and release notes
  maintainers: claim service cores

 MAINTAINERS                                        |   7 +
 doc/api/doxy-api-index.md                          |   1 +
 doc/guides/eventdevs/sw.rst                        |   4 +-
 doc/guides/prog_guide/index.rst                    |   1 +
 doc/guides/prog_guide/service_cores.rst            |  81 +++
 doc/guides/rel_notes/release_17_08.rst             |   8 +
 drivers/event/sw/sw_evdev.c                        |  32 +
 drivers/event/sw/sw_evdev.h                        |   3 +
 lib/librte_eal/bsdapp/eal/Makefile                 |   1 +
 lib/librte_eal/bsdapp/eal/eal.c                    |  23 +
 lib/librte_eal/bsdapp/eal/rte_eal_version.map      |  22 +
 lib/librte_eal/common/Makefile                     |   1 +
 lib/librte_eal/common/eal_common_lcore.c           |   1 +
 lib/librte_eal/common/eal_common_options.c         |  90 ++-
 lib/librte_eal/common/include/rte_eal.h            |   4 +
 lib/librte_eal/common/include/rte_lcore.h          |   3 +-
 lib/librte_eal/common/include/rte_service.h        | 383 ++++++++++++
 .../common/include/rte_service_private.h           | 140 +++++
 lib/librte_eal/common/rte_service.c                | 687 +++++++++++++++++++++
 lib/librte_eal/linuxapp/eal/Makefile               |   1 +
 lib/librte_eal/linuxapp/eal/eal.c                  |  23 +
 lib/librte_eal/linuxapp/eal/eal_thread.c           |   9 +-
 lib/librte_eal/linuxapp/eal/rte_eal_version.map    |  22 +
 test/test/Makefile                                 |   2 +
 test/test/test_service_cores.c                     | 538 ++++++++++++++++
 25 files changed, 2083 insertions(+), 4 deletions(-)
 create mode 100644 doc/guides/prog_guide/service_cores.rst
 create mode 100644 lib/librte_eal/common/include/rte_service.h
 create mode 100644 lib/librte_eal/common/include/rte_service_private.h
 create mode 100644 lib/librte_eal/common/rte_service.c
 create mode 100644 test/test/test_service_cores.c

-- 
2.7.4

[PATCH v4 1/7] service cores: header and implementation

[Harry van Haaren]

Add header files, update .map files with new service
functions, and add the service header to the doxygen
for building.

This service header API allows DPDK to use services as
a concept of something that requires CPU cycles. An example
is a PMD that runs in software to schedule events, where a
hardware version exists that does not require a CPU.

The code presented here is based on an initial RFC:
http://dpdk.org/ml/archives/dev/2017-May/065207.html
This was then reworked, and RFC v2 with the changes posted:
http://dpdk.org/ml/archives/dev/2017-June/067194.html

This is the fourth iteration of the service core concept,
with 2 RFCs and this being v2 of the implementation.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

---

v4:
- Fixed (unsigned) checkpatch error
- Fixed misleading-indentation/if { } brackets (checkpatch/Jerin)
- Fixed set function argument to be "enable" instead of "enabled" (Jerin)
- Improve doxygen comment for size of array in rte_service_core_list (Jerin)
- Fixed typos (Jerin)
- Optimized atomic clear after running service (Jerin)
- Added smp_rmb() at end of loop to re-load runstate / mapping (Jerin)
- Fix issue with lcore role not being adhered to (Jerin)
- Add experimental warnings for all service core functions (Thomas)
- Moved service core functions into EXPERIMENTAL section of .map (Thomas)
- Improve documentation of rte_service_lcore_reset_all() (Harry)

v3:
- None.

v2:
Thanks Jerin for review - below a list your suggested changes;
- Doxygen rename to "service cores" for consistency
- use lcore instead of core for function names
- Fix about 10 typos / seplling msitakse ;)
- Dix doxygen /** comments for functions
- Doxygen @param[out] improvements
- int8_t for socket_id to ordinary int
- Rename MACROS for readability
- Align structs to cache lines
- Allocate fastpath-used data from hugepages
- Added/fixed memory barriers for multi-core scheduling
- Add const to variables, and hoist above loop
- Optimize cmpset atomic if MT_SAFE or only one core mapped
- Statistics collection only when requested
- Add error check for array pointer
- Remove panic() calls from library
- Fix TODO notes from previous patchset

There are also some other changes;
- Checkpatch issues fixed
- .map file updates
- Add rte_service_get_by_name() function
---
 doc/api/doxy-api-index.md                          |   1 +
 lib/librte_eal/bsdapp/eal/Makefile                 |   1 +
 lib/librte_eal/bsdapp/eal/rte_eal_version.map      |  22 +
 lib/librte_eal/common/Makefile                     |   1 +
 lib/librte_eal/common/eal_common_lcore.c           |   1 +
 lib/librte_eal/common/include/rte_eal.h            |   4 +
 lib/librte_eal/common/include/rte_lcore.h          |   3 +-
 lib/librte_eal/common/include/rte_service.h        | 383 ++++++++++++
 .../common/include/rte_service_private.h           | 140 +++++
 lib/librte_eal/common/rte_service.c                | 687 +++++++++++++++++++++
 lib/librte_eal/linuxapp/eal/Makefile               |   1 +
 lib/librte_eal/linuxapp/eal/eal_thread.c           |   9 +-
 lib/librte_eal/linuxapp/eal/rte_eal_version.map    |  22 +
 13 files changed, 1273 insertions(+), 2 deletions(-)
 create mode 100644 lib/librte_eal/common/include/rte_service.h
 create mode 100644 lib/librte_eal/common/include/rte_service_private.h
 create mode 100644 lib/librte_eal/common/rte_service.c

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index 3b83288..e2abdf4 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -159,6 +159,7 @@ There are many libraries, so their headers may be grouped by topics:
   [common]             (@ref rte_common.h),
   [ABI compat]         (@ref rte_compat.h),
   [keepalive]          (@ref rte_keepalive.h),
+  [service cores]      (@ref rte_service.h),
   [device metrics]     (@ref rte_metrics.h),
   [bitrate statistics] (@ref rte_bitrate.h),
   [latency statistics] (@ref rte_latencystats.h),
diff --git a/lib/librte_eal/bsdapp/eal/Makefile b/lib/librte_eal/bsdapp/eal/Makefile
index a0f9950..05517a2 100644
--- a/lib/librte_eal/bsdapp/eal/Makefile
+++ b/lib/librte_eal/bsdapp/eal/Makefile
@@ -87,6 +87,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_malloc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_elem.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_heap.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_keepalive.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_service.c
 
 # from arch dir
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_cpuflags.c
diff --git a/lib/librte_eal/bsdapp/eal/rte_eal_version.map b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
index 0295ea9..130b2c0 100644
--- a/lib/librte_eal/bsdapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
@@ -208,5 +208,27 @@ EXPERIMENTAL {
 
 	rte_eal_hotplug_add;
 	rte_eal_hotplug_remove;
+	rte_service_disable_on_lcore;
+	rte_service_dump;
+	rte_service_enable_on_lcore;
+	rte_service_get_by_id;
+	rte_service_get_by_name;
+	rte_service_get_count;
+	rte_service_get_enabled_on_lcore;
+	rte_service_is_running;
+	rte_service_lcore_add;
+	rte_service_lcore_count;
+	rte_service_lcore_del;
+	rte_service_lcore_list;
+	rte_service_lcore_reset_all;
+	rte_service_lcore_start;
+	rte_service_lcore_stop;
+	rte_service_probe_capability;
+	rte_service_register;
+	rte_service_reset;
+	rte_service_set_stats_enable;
+	rte_service_start;
+	rte_service_stop;
+	rte_service_unregister;
 
 } DPDK_17.08;
diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
index f2fe052..942f03c 100644
--- a/lib/librte_eal/common/Makefile
+++ b/lib/librte_eal/common/Makefile
@@ -41,6 +41,7 @@ INC += rte_eal_memconfig.h rte_malloc_heap.h
 INC += rte_hexdump.h rte_devargs.h rte_bus.h rte_dev.h rte_vdev.h
 INC += rte_pci_dev_feature_defs.h rte_pci_dev_features.h
 INC += rte_malloc.h rte_keepalive.h rte_time.h
+INC += rte_service.h rte_service_private.h
 
 GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h
 GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h
diff --git a/lib/librte_eal/common/eal_common_lcore.c b/lib/librte_eal/common/eal_common_lcore.c
index 84fa0cb..0db1555 100644
--- a/lib/librte_eal/common/eal_common_lcore.c
+++ b/lib/librte_eal/common/eal_common_lcore.c
@@ -81,6 +81,7 @@ rte_eal_cpu_init(void)
 
 		/* By default, each detected core is enabled */
 		config->lcore_role[lcore_id] = ROLE_RTE;
+		lcore_config[lcore_id].core_role = ROLE_RTE;
 		lcore_config[lcore_id].core_id = eal_cpu_core_id(lcore_id);
 		lcore_config[lcore_id].socket_id = eal_cpu_socket_id(lcore_id);
 		if (lcore_config[lcore_id].socket_id >= RTE_MAX_NUMA_NODES) {
diff --git a/lib/librte_eal/common/include/rte_eal.h b/lib/librte_eal/common/include/rte_eal.h
index abf020b..4dd0518 100644
--- a/lib/librte_eal/common/include/rte_eal.h
+++ b/lib/librte_eal/common/include/rte_eal.h
@@ -61,6 +61,7 @@ extern "C" {
 enum rte_lcore_role_t {
 	ROLE_RTE,
 	ROLE_OFF,
+	ROLE_SERVICE,
 };
 
 /**
@@ -80,6 +81,7 @@ enum rte_proc_type_t {
 struct rte_config {
 	uint32_t master_lcore;       /**< Id of the master lcore */
 	uint32_t lcore_count;        /**< Number of available logical cores. */
+	uint32_t service_lcore_count;/**< Number of available service cores. */
 	enum rte_lcore_role_t lcore_role[RTE_MAX_LCORE]; /**< State of cores. */
 
 	/** Primary or secondary configuration */
@@ -185,6 +187,8 @@ int rte_eal_iopl_init(void);
  *
  *     EPROTO indicates that the PCI bus is either not present, or is not
  *            readable by the eal.
+ *
+ *     ENOEXEC indicates that a service core failed to launch successfully.
  */
 int rte_eal_init(int argc, char **argv);
 
diff --git a/lib/librte_eal/common/include/rte_lcore.h b/lib/librte_eal/common/include/rte_lcore.h
index fe7b586..50e0d0f 100644
--- a/lib/librte_eal/common/include/rte_lcore.h
+++ b/lib/librte_eal/common/include/rte_lcore.h
@@ -73,6 +73,7 @@ struct lcore_config {
 	unsigned core_id;          /**< core number on socket for this lcore */
 	int core_index;            /**< relative index, starting from 0 */
 	rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
+	uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
 };
 
 /**
@@ -175,7 +176,7 @@ rte_lcore_is_enabled(unsigned lcore_id)
 	struct rte_config *cfg = rte_eal_get_configuration();
 	if (lcore_id >= RTE_MAX_LCORE)
 		return 0;
-	return cfg->lcore_role[lcore_id] != ROLE_OFF;
+	return cfg->lcore_role[lcore_id] == ROLE_RTE;
 }
 
 /**
diff --git a/lib/librte_eal/common/include/rte_service.h b/lib/librte_eal/common/include/rte_service.h
new file mode 100644
index 0000000..7588a41
--- /dev/null
+++ b/lib/librte_eal/common/include/rte_service.h
@@ -0,0 +1,383 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_SERVICE_H_
+#define _RTE_SERVICE_H_
+
+/**
+ * @file
+ *
+ * Service functions
+ *
+ * The service functionality provided by this header allows a DPDK component
+ * to indicate that it requires a function call in order for it to perform
+ * its processing.
+ *
+ * An example usage of this functionality would be a component that registers
+ * a service to perform a particular packet processing duty: for example the
+ * eventdev software PMD. At startup the application requests all services
+ * that have been registered, and the cores in the service-coremask run the
+ * required services. The EAL removes these number of cores from the available
+ * runtime cores, and dedicates them to performing service-core workloads. The
+ * application has access to the remaining lcores as normal.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include<stdio.h>
+#include <stdint.h>
+#include <sys/queue.h>
+
+#include <rte_lcore.h>
+
+/* forward declaration only. Definition in rte_service_private.h */
+struct rte_service_spec;
+
+#define RTE_SERVICE_NAME_MAX 32
+
+/* Capabilities of a service.
+ *
+ * Use the *rte_service_probe_capability* function to check if a service is
+ * capable of a specific capability.
+ */
+/** When set, the service is capable of having multiple threads run it at the
+ *  same time.
+ */
+#define RTE_SERVICE_CAP_MT_SAFE (1 << 0)
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ *  Return the number of services registered.
+ *
+ * The number of services registered can be passed to *rte_service_get_by_id*,
+ * enabling the application to retrieve the specification of each service.
+ *
+ * @return The number of services registered.
+ */
+uint32_t rte_service_get_count(void);
+
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Return the specification of a service by integer id.
+ *
+ * This function provides the specification of a service. This can be used by
+ * the application to understand what the service represents. The service
+ * must not be modified by the application directly, only passed to the various
+ * rte_service_* functions.
+ *
+ * @param id The integer id of the service to retrieve
+ * @retval non-zero A valid pointer to the service_spec
+ * @retval NULL Invalid *id* provided.
+ */
+struct rte_service_spec *rte_service_get_by_id(uint32_t id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Return the specification of a service by name.
+ *
+ * This function provides the specification of a service using the service name
+ * as lookup key. This can be used by the application to understand what the
+ * service represents. The service must not be modified by the application
+ * directly, only passed to the various rte_service_* functions.
+ *
+ * @param name The name of the service to retrieve
+ * @retval non-zero A valid pointer to the service_spec
+ * @retval NULL Invalid *name* provided.
+ */
+struct rte_service_spec *rte_service_get_by_name(const char *name);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Return the name of the service.
+ *
+ * @return A pointer to the name of the service. The returned pointer remains
+ *         in ownership of the service, and the application must not free it.
+ */
+const char *rte_service_get_name(const struct rte_service_spec *service);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Check if a service has a specific capability.
+ *
+ * This function returns if *service* has implements *capability*.
+ * See RTE_SERVICE_CAP_* defines for a list of valid capabilities.
+ * @retval 1 Capability supported by this service instance
+ * @retval 0 Capability not supported by this service instance
+ */
+int32_t rte_service_probe_capability(const struct rte_service_spec *service,
+				     uint32_t capability);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Enable a core to run a service.
+ *
+ * Each core can be added or removed from running specific services. This
+ * functions adds *lcore* to the set of cores that will run *service*.
+ *
+ * If multiple cores are enabled on a service, an atomic is used to ensure that
+ * only one cores runs the service at a time. The exception to this is when
+ * a service indicates that it is multi-thread safe by setting the capability
+ * called RTE_SERVICE_CAP_MT_SAFE. With the multi-thread safe capability set,
+ * the service function can be run on multiple threads at the same time.
+ *
+ * @retval 0 lcore added successfully
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_enable_on_lcore(struct rte_service_spec *service,
+				   uint32_t lcore);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Disable a core to run a service.
+ *
+ * Each core can be added or removed from running specific services. This
+ * functions removes *lcore* to the set of cores that will run *service*.
+ *
+ * @retval 0 Lcore removed successfully
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_disable_on_lcore(struct rte_service_spec *service,
+				   uint32_t lcore);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Return if an lcore is enabled for the service.
+ *
+ * This function allows the application to query if *lcore* is currently set to
+ * run *service*.
+ *
+ * @retval 1 Lcore enabled on this lcore
+ * @retval 0 Lcore disabled on this lcore
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_get_enabled_on_lcore(struct rte_service_spec *service,
+					uint32_t lcore);
+
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Enable *service* to run.
+ *
+ * This function switches on a service during runtime.
+ * @retval 0 The service was successfully started
+ */
+int32_t rte_service_start(struct rte_service_spec *service);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Disable *service*.
+ *
+ * Switch off a service, so it is not run until it is *rte_service_start* is
+ * called on it.
+ * @retval 0 Service successfully switched off
+ */
+int32_t rte_service_stop(struct rte_service_spec *service);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Returns if *service* is currently running.
+ *
+ * This function returns true if the service has been started using
+ * *rte_service_start*, AND a service core is mapped to the service. This
+ * function can be used to ensure that the service will be run.
+ *
+ * @retval 1 Service is currently running, and has a service lcore mapped
+ * @retval 0 Service is currently stopped, or no service lcore is mapped
+ * @retval -EINVAL Invalid service pointer provided
+ */
+int32_t rte_service_is_running(const struct rte_service_spec *service);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Start a service core.
+ *
+ * Starting a core makes the core begin polling. Any services assigned to it
+ * will be run as fast as possible.
+ *
+ * @retval 0 Success
+ * @retval -EINVAL Failed to start core. The *lcore_id* passed in is not
+ *          currently assigned to be a service core.
+ */
+int32_t rte_service_lcore_start(uint32_t lcore_id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Stop a service core.
+ *
+ * Stopping a core makes the core become idle, but remains  assigned as a
+ * service core.
+ *
+ * @retval 0 Success
+ * @retval -EINVAL Invalid *lcore_id* provided
+ * @retval -EALREADY Already stopped core
+ * @retval -EBUSY Failed to stop core, as it would cause a service to not
+ *          be run, as this is the only core currently running the service.
+ *          The application must stop the service first, and then stop the
+ *          lcore.
+ */
+int32_t rte_service_lcore_stop(uint32_t lcore_id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Adds lcore to the list of service cores.
+ *
+ * This functions can be used at runtime in order to modify the service core
+ * mask.
+ *
+ * @retval 0 Success
+ * @retval -EBUSY lcore is busy, and not available for service core duty
+ * @retval -EALREADY lcore is already added to the service core list
+ * @retval -EINVAL Invalid lcore provided
+ */
+int32_t rte_service_lcore_add(uint32_t lcore);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Removes lcore from the list of service cores.
+ *
+ * This can fail if the core is not stopped, see *rte_service_core_stop*.
+ *
+ * @retval 0 Success
+ * @retval -EBUSY Lcore is not stopped, stop service core before removing.
+ * @retval -EINVAL failed to add lcore to service core mask.
+ */
+int32_t rte_service_lcore_del(uint32_t lcore);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Retrieve the number of service cores currently available.
+ *
+ * This function returns the integer count of service cores available. The
+ * service core count can be used in mapping logic when creating mappings
+ * from service cores to services.
+ *
+ * See *rte_service_lcore_list* for details on retrieving the lcore_id of each
+ * service core.
+ *
+ * @return The number of service cores currently configured.
+ */
+int32_t rte_service_lcore_count(void);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Resets all service core mappings. This does not remove the service cores
+ * from duty, just unmaps all services / cores, and stops() the service cores.
+ * The runstate of services is not modified.
+ *
+ * @retval 0 Success
+ */
+int32_t rte_service_lcore_reset_all(void);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Enable or disable statistics collection.
+ *
+ * This function enables per core, per-service cycle count collection.
+ * @param enabled Zero to turn off statistics collection, non-zero to enable.
+ */
+void rte_service_set_stats_enable(int enable);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Retrieve the list of currently enabled service cores.
+ *
+ * This function fills in an application supplied array, with each element
+ * indicating the lcore_id of a service core.
+ *
+ * Adding and removing service cores can be performed using
+ * *rte_service_lcore_add* and *rte_service_lcore_del*.
+ * @param [out] array An array of at least *rte_service_lcore_count* items.
+ *              If statically allocating the buffer, use RTE_MAX_LCORE.
+ * @param [out] The size of *array*.
+ * @retval >=0 Number of service cores that have been populated in the array
+ * @retval -ENOMEM The provided array is not large enough to fill in the
+ *          service core list. No items have been populated, call this function
+ *          with a size of at least *rte_service_core_count* items.
+ */
+int32_t rte_service_lcore_list(uint32_t array[], uint32_t n);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Dumps any information available about the service. If service is NULL,
+ * dumps info for all services.
+ */
+int32_t rte_service_dump(FILE *f, struct rte_service_spec *service);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* _RTE_SERVICE_H_ */
diff --git a/lib/librte_eal/common/include/rte_service_private.h b/lib/librte_eal/common/include/rte_service_private.h
new file mode 100644
index 0000000..e726108
--- /dev/null
+++ b/lib/librte_eal/common/include/rte_service_private.h
@@ -0,0 +1,140 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_SERVICE_PRIVATE_H_
+#define _RTE_SERVICE_PRIVATE_H_
+
+/* This file specifies the internal service specification.
+ * Include this file if you are writing a component that requires CPU cycles to
+ * operate, and you wish to run the component using service cores
+ */
+
+#include <rte_service.h>
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Signature of callback function to run a service.
+ */
+typedef int32_t (*rte_service_func)(void *args);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * The specification of a service.
+ *
+ * This struct contains metadata about the service itself, the callback
+ * function to run one iteration of the service, a userdata pointer, flags etc.
+ */
+struct rte_service_spec {
+	/** The name of the service. This should be used by the application to
+	 * understand what purpose this service provides.
+	 */
+	char name[RTE_SERVICE_NAME_MAX];
+	/** The callback to invoke to run one iteration of the service. */
+	rte_service_func callback;
+	/** The userdata pointer provided to the service callback. */
+	void *callback_userdata;
+	/** Flags to indicate the capabilities of this service. See defines in
+	 * the public header file for values of RTE_SERVICE_CAP_*
+	 */
+	uint32_t capabilities;
+	/** NUMA socket ID that this service is affinitized to */
+	int socket_id;
+};
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Register a new service.
+ *
+ * A service represents a component that the requires CPU time periodically to
+ * achieve its purpose.
+ *
+ * For example the eventdev SW PMD requires CPU cycles to perform its
+ * scheduling. This can be achieved by registering it as a service, and the
+ * application can then assign CPU resources to it using
+ * *rte_service_set_coremask*.
+ *
+ * @param spec The specification of the service to register
+ * @retval 0 Successfully registered the service.
+ *         -EINVAL Attempted to register an invalid service (eg, no callback
+ *         set)
+ */
+int32_t rte_service_register(const struct rte_service_spec *spec);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Unregister a service.
+ *
+ * The service being removed must be stopped before calling this function.
+ *
+ * @retval 0 The service was successfully unregistered.
+ * @retval -EBUSY The service is currently running, stop the service before
+ *          calling unregister. No action has been taken.
+ */
+int32_t rte_service_unregister(struct rte_service_spec *service);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Private function to allow EAL to initialized default mappings.
+ *
+ * This function iterates all the services, and maps then to the available
+ * cores. Based on the capabilities of the services, they are set to run on the
+ * available cores in a round-robin manner.
+ *
+ * @retval 0 Success
+ */
+int32_t rte_service_set_default_mapping(void);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Initialize the service library.
+ *
+ * In order to use the service library, it must be initialized. EAL initializes
+ * the library at startup.
+ *
+ * @retval 0 Success
+ * @retval -EALREADY Service library is already initialized
+ */
+int32_t rte_service_init(void);
+
+#endif /* _RTE_SERVICE_PRIVATE_H_ */
diff --git a/lib/librte_eal/common/rte_service.c b/lib/librte_eal/common/rte_service.c
new file mode 100644
index 0000000..223ca22
--- /dev/null
+++ b/lib/librte_eal/common/rte_service.c
@@ -0,0 +1,687 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_service.h>
+#include "include/rte_service_private.h"
+
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_common.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_atomic.h>
+#include <rte_memory.h>
+#include <rte_malloc.h>
+
+#define RTE_SERVICE_NUM_MAX 64
+
+#define SERVICE_F_REGISTERED 0
+
+/* runstates for services and lcores, denoting if they are active or not */
+#define RUNSTATE_STOPPED 0
+#define RUNSTATE_RUNNING 1
+
+/* internal representation of a service */
+struct rte_service_spec_impl {
+	/* public part of the struct */
+	struct rte_service_spec spec;
+
+	/* atomic lock that when set indicates a service core is currently
+	 * running this service callback. When not set, a core may take the
+	 * lock and then run the service callback.
+	 */
+	rte_atomic32_t execute_lock;
+
+	/* API set/get-able variables */
+	int32_t runstate;
+	uint8_t internal_flags;
+
+	/* per service statistics */
+	uint32_t num_mapped_cores;
+	uint64_t calls;
+	uint64_t cycles_spent;
+} __rte_cache_aligned;
+
+/* the internal values of a service core */
+struct core_state {
+	/* map of services IDs are run on this core */
+	uint64_t service_mask;
+	uint8_t runstate; /* running or stopped */
+	uint8_t is_service_core; /* set if core is currently a service core */
+	uint8_t collect_statistics; /* if set, measure cycle counts */
+
+	/* extreme statistics */
+	uint64_t calls_per_service[RTE_SERVICE_NUM_MAX];
+} __rte_cache_aligned;
+
+static uint32_t rte_service_count;
+static struct rte_service_spec_impl *rte_services;
+static struct core_state *cores_state;
+static uint32_t rte_service_library_initialized;
+
+int32_t rte_service_init(void)
+{
+	if (rte_service_library_initialized) {
+		printf("service library init() called, init flag %d\n",
+			rte_service_library_initialized);
+		return -EALREADY;
+	}
+
+	rte_services = rte_calloc("rte_services", RTE_SERVICE_NUM_MAX,
+			sizeof(struct rte_service_spec_impl),
+			RTE_CACHE_LINE_SIZE);
+	if (!rte_services) {
+		printf("error allocating rte services array\n");
+		return -ENOMEM;
+	}
+
+	cores_state = rte_calloc("rte_service_core_states", RTE_MAX_LCORE,
+			sizeof(struct core_state), RTE_CACHE_LINE_SIZE);
+	if (!cores_state) {
+		printf("error allocating core states array\n");
+		return -ENOMEM;
+	}
+
+	int i;
+	int count = 0;
+	struct rte_config *cfg = rte_eal_get_configuration();
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		if (lcore_config[i].core_role == ROLE_SERVICE) {
+			if ((unsigned int)i == cfg->master_lcore)
+				continue;
+			rte_service_lcore_add(i);
+			count++;
+		}
+	}
+
+	rte_service_library_initialized = 1;
+	return 0;
+}
+
+void rte_service_set_stats_enable(int enabled)
+{
+	uint32_t i;
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		cores_state[i].collect_statistics = enabled;
+}
+
+/* returns 1 if service is registered and has not been unregistered
+ * Returns 0 if service never registered, or has been unregistered
+ */
+static inline int
+service_valid(uint32_t id) {
+	return !!(rte_services[id].internal_flags &
+		 (1 << SERVICE_F_REGISTERED));
+}
+
+uint32_t
+rte_service_get_count(void)
+{
+	return rte_service_count;
+}
+
+struct rte_service_spec *
+rte_service_get_by_id(uint32_t id)
+{
+	struct rte_service_spec *service = NULL;
+	if (id < rte_service_count)
+		service = (struct rte_service_spec *)&rte_services[id];
+
+	return service;
+}
+
+struct rte_service_spec *rte_service_get_by_name(const char *name)
+{
+	struct rte_service_spec *service = NULL;
+	int i;
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (service_valid(i) &&
+				strcmp(name, rte_services[i].spec.name) == 0) {
+			service = (struct rte_service_spec *)&rte_services[i];
+			break;
+		}
+	}
+
+	return service;
+}
+
+const char *
+rte_service_get_name(const struct rte_service_spec *service)
+{
+	return service->name;
+}
+
+int32_t
+rte_service_probe_capability(const struct rte_service_spec *service,
+			     uint32_t capability)
+{
+	return service->capabilities & capability;
+}
+
+int32_t
+rte_service_is_running(const struct rte_service_spec *spec)
+{
+	const struct rte_service_spec_impl *impl =
+		(const struct rte_service_spec_impl *)spec;
+	if (!impl)
+		return -EINVAL;
+
+	return (impl->runstate == RUNSTATE_RUNNING) &&
+		(impl->num_mapped_cores > 0);
+}
+
+int32_t
+rte_service_register(const struct rte_service_spec *spec)
+{
+	uint32_t i;
+	int32_t free_slot = -1;
+
+	if (spec->callback == NULL || strlen(spec->name) == 0)
+		return -EINVAL;
+
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (!service_valid(i)) {
+			free_slot = i;
+			break;
+		}
+	}
+
+	if ((free_slot < 0) || (i == RTE_SERVICE_NUM_MAX))
+		return -ENOSPC;
+
+	struct rte_service_spec_impl *s = &rte_services[free_slot];
+	s->spec = *spec;
+	s->internal_flags |= (1 << SERVICE_F_REGISTERED);
+
+	rte_smp_wmb();
+	rte_service_count++;
+
+	return 0;
+}
+
+int32_t
+rte_service_unregister(struct rte_service_spec *spec)
+{
+	struct rte_service_spec_impl *s = NULL;
+	struct rte_service_spec_impl *spec_impl =
+		(struct rte_service_spec_impl *)spec;
+
+	uint32_t i;
+	uint32_t service_id;
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (&rte_services[i] == spec_impl) {
+			s = spec_impl;
+			service_id = i;
+			break;
+		}
+	}
+
+	if (!s)
+		return -EINVAL;
+
+	rte_service_count--;
+	rte_smp_wmb();
+
+	s->internal_flags &= ~(1 << SERVICE_F_REGISTERED);
+
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		cores_state[i].service_mask &= ~(1 << service_id);
+
+	memset(&rte_services[service_id], 0,
+			sizeof(struct rte_service_spec_impl));
+
+	return 0;
+}
+
+int32_t
+rte_service_start(struct rte_service_spec *service)
+{
+	struct rte_service_spec_impl *s =
+		(struct rte_service_spec_impl *)service;
+	s->runstate = RUNSTATE_RUNNING;
+	rte_smp_wmb();
+	return 0;
+}
+
+int32_t
+rte_service_stop(struct rte_service_spec *service)
+{
+	struct rte_service_spec_impl *s =
+		(struct rte_service_spec_impl *)service;
+	s->runstate = RUNSTATE_STOPPED;
+	rte_smp_wmb();
+	return 0;
+}
+
+static int32_t
+rte_service_runner_func(void *arg)
+{
+	RTE_SET_USED(arg);
+	uint32_t i;
+	const int lcore = rte_lcore_id();
+	struct core_state *cs = &cores_state[lcore];
+
+	while (cores_state[lcore].runstate == RUNSTATE_RUNNING) {
+		const uint64_t service_mask = cs->service_mask;
+		for (i = 0; i < rte_service_count; i++) {
+			struct rte_service_spec_impl *s = &rte_services[i];
+			if (s->runstate != RUNSTATE_RUNNING ||
+					!(service_mask & (1 << i)))
+				continue;
+
+			/* check if this is the only core mapped, else use
+			 * atomic to serialize cores mapped to this service
+			 */
+			uint32_t *lock = (uint32_t *)&s->execute_lock;
+			if ((s->spec.capabilities & RTE_SERVICE_CAP_MT_SAFE) ||
+					(s->num_mapped_cores == 1 ||
+					rte_atomic32_cmpset(lock, 0, 1))) {
+				void *userdata = s->spec.callback_userdata;
+
+				if (cs->collect_statistics) {
+					uint64_t start = rte_rdtsc();
+					s->spec.callback(userdata);
+					uint64_t end = rte_rdtsc();
+					s->cycles_spent += end - start;
+					cs->calls_per_service[i]++;
+					s->calls++;
+				} else
+					s->spec.callback(userdata);
+
+				if ((s->spec.capabilities &
+						RTE_SERVICE_CAP_MT_SAFE) == 0 &&
+						s->num_mapped_cores > 1)
+					rte_atomic32_clear(&s->execute_lock);
+			}
+		}
+
+		rte_smp_rmb();
+	}
+
+	lcore_config[lcore].state = WAIT;
+
+	return 0;
+}
+
+int32_t
+rte_service_lcore_count(void)
+{
+	int32_t count = 0;
+	uint32_t i;
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		count += cores_state[i].is_service_core;
+	return count;
+}
+
+int32_t
+rte_service_lcore_list(uint32_t array[], uint32_t n)
+{
+	uint32_t count = rte_service_lcore_count();
+	if (count > n)
+		return -ENOMEM;
+
+	if (!array)
+		return -EINVAL;
+
+	uint32_t i;
+	uint32_t idx = 0;
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct core_state *cs = &cores_state[i];
+		if (cs->is_service_core) {
+			array[idx] = i;
+			idx++;
+		}
+	}
+
+	return count;
+}
+
+int32_t
+rte_service_set_default_mapping(void)
+{
+	/* create a default mapping from cores to services, then start the
+	 * services to make them transparent to unaware applications.
+	 */
+	uint32_t i;
+	int ret;
+	uint32_t count = rte_service_get_count();
+
+	int32_t lcore_iter = 0;
+	uint32_t ids[RTE_MAX_LCORE];
+	int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE);
+
+	for (i = 0; i < count; i++) {
+		struct rte_service_spec *s = rte_service_get_by_id(i);
+		if (!s)
+			return -EINVAL;
+
+		/* if no lcores available as services cores, don't setup map.
+		 * This means app logic must add cores, and setup mappings
+		 */
+		if (lcore_count > 0) {
+			/* do 1:1 core mapping here, with each service getting
+			 * assigned a single core by default. Adding multiple
+			 * services should multiplex to a single core, or 1:1
+			 * if services == cores
+			 */
+			ret = rte_service_enable_on_lcore(s, ids[lcore_iter]);
+			if (ret)
+				return -ENODEV;
+		}
+
+		lcore_iter++;
+		if (lcore_iter >= lcore_count)
+			lcore_iter = 0;
+
+		ret = rte_service_start(s);
+		if (ret)
+			return -ENOEXEC;
+	}
+
+	return 0;
+}
+
+static int32_t
+service_update(struct rte_service_spec *service, uint32_t lcore,
+		uint32_t *set, uint32_t *enabled)
+{
+	uint32_t i;
+	int32_t sid = -1;
+
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if ((struct rte_service_spec *)&rte_services[i] == service &&
+				service_valid(i)) {
+			sid = i;
+			break;
+		}
+	}
+
+	if (sid == -1 || lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	if (!cores_state[lcore].is_service_core)
+		return -EINVAL;
+
+	if (set) {
+		if (*set) {
+			cores_state[lcore].service_mask |=  (1 << sid);
+			rte_services[sid].num_mapped_cores++;
+		} else {
+			cores_state[lcore].service_mask &= ~(1 << sid);
+			rte_services[sid].num_mapped_cores--;
+		}
+	}
+
+	if (enabled)
+		*enabled = (cores_state[lcore].service_mask & (1 << sid));
+
+	rte_smp_wmb();
+
+	return 0;
+}
+
+int32_t rte_service_get_enabled_on_lcore(struct rte_service_spec *service,
+					uint32_t lcore)
+{
+	uint32_t enabled;
+	int ret = service_update(service, lcore, 0, &enabled);
+	if (ret == 0)
+		return enabled;
+	return -EINVAL;
+}
+
+int32_t
+rte_service_enable_on_lcore(struct rte_service_spec *service, uint32_t lcore)
+{
+	uint32_t on = 1;
+	return service_update(service, lcore, &on, 0);
+}
+
+int32_t
+rte_service_disable_on_lcore(struct rte_service_spec *service, uint32_t lcore)
+{
+	uint32_t off = 0;
+	return service_update(service, lcore, &off, 0);
+}
+
+int32_t rte_service_lcore_reset_all(void)
+{
+	/* loop over cores, reset all to mask 0 */
+	uint32_t i;
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		cores_state[i].service_mask = 0;
+		cores_state[i].is_service_core = 0;
+		cores_state[i].runstate = RUNSTATE_STOPPED;
+	}
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++)
+		rte_services[i].num_mapped_cores = 0;
+
+	rte_smp_wmb();
+
+	return 0;
+}
+
+static void
+set_lcore_state(uint32_t lcore, int32_t state)
+{
+	/* mark core state in hugepage backed config */
+	struct rte_config *cfg = rte_eal_get_configuration();
+	cfg->lcore_role[lcore] = state;
+
+	/* mark state in process local lcore_config */
+	lcore_config[lcore].core_role = state;
+
+	/* update per-lcore optimized state tracking */
+	cores_state[lcore].is_service_core = (state == ROLE_SERVICE);
+}
+
+int32_t
+rte_service_lcore_add(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+	if (cores_state[lcore].is_service_core)
+		return -EALREADY;
+
+	set_lcore_state(lcore, ROLE_SERVICE);
+
+	/* ensure that after adding a core the mask and state are defaults */
+	cores_state[lcore].service_mask = 0;
+	cores_state[lcore].runstate = RUNSTATE_STOPPED;
+
+	return 0;
+}
+
+int32_t
+rte_service_lcore_del(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	struct core_state *cs = &cores_state[lcore];
+	if (!cs->is_service_core)
+		return -EINVAL;
+
+	if (cs->runstate != RUNSTATE_STOPPED)
+		return -EBUSY;
+
+	set_lcore_state(lcore, ROLE_RTE);
+
+	return 0;
+}
+
+int32_t
+rte_service_lcore_start(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	struct core_state *cs = &cores_state[lcore];
+	if (!cs->is_service_core)
+		return -EINVAL;
+
+	if (cs->runstate == RUNSTATE_RUNNING)
+		return -EALREADY;
+
+	/* set core to run state first, and then launch otherwise it will
+	 * return immediately as runstate keeps it in the service poll loop
+	 */
+	cores_state[lcore].runstate = RUNSTATE_RUNNING;
+
+	int ret = rte_eal_remote_launch(rte_service_runner_func, 0, lcore);
+	/* returns -EBUSY if the core is already launched, 0 on success */
+	return ret;
+}
+
+int32_t
+rte_service_lcore_stop(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	if (cores_state[lcore].runstate == RUNSTATE_STOPPED)
+		return -EALREADY;
+
+	uint32_t i;
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		int32_t enabled = cores_state[i].service_mask & (1 << i);
+		int32_t service_running = rte_services[i].runstate !=
+						RUNSTATE_STOPPED;
+		int32_t only_core = rte_services[i].num_mapped_cores == 1;
+
+		/* if the core is mapped, and the service is running, and this
+		 * is the only core that is mapped, the service would cease to
+		 * run if this core stopped, so fail instead.
+		 */
+		if (enabled && service_running && only_core)
+			return -EBUSY;
+	}
+
+	cores_state[lcore].runstate = RUNSTATE_STOPPED;
+
+	return 0;
+}
+
+static void
+rte_service_dump_one(FILE *f, struct rte_service_spec_impl *s,
+		     uint64_t all_cycles, uint32_t reset)
+{
+	/* avoid divide by zero */
+	if (all_cycles == 0)
+		all_cycles = 1;
+
+	int calls = 1;
+	if (s->calls != 0)
+		calls = s->calls;
+
+	float cycles_pct = (((float)s->cycles_spent) / all_cycles) * 100.f;
+	fprintf(f,
+			"  %s : %0.1f %%\tcalls %"PRIu64"\tcycles %"
+			PRIu64"\tavg: %"PRIu64"\n",
+			s->spec.name, cycles_pct, s->calls, s->cycles_spent,
+			s->cycles_spent / calls);
+
+	if (reset) {
+		s->cycles_spent = 0;
+		s->calls = 0;
+	}
+}
+
+static void
+service_dump_calls_per_lcore(FILE *f, uint32_t lcore, uint32_t reset)
+{
+	uint32_t i;
+	struct core_state *cs = &cores_state[lcore];
+
+	fprintf(f, "%02d\t", lcore);
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (!service_valid(i))
+			continue;
+		fprintf(f, "%"PRIu64"\t", cs->calls_per_service[i]);
+		if (reset)
+			cs->calls_per_service[i] = 0;
+	}
+	fprintf(f, "\n");
+}
+
+int32_t rte_service_dump(FILE *f, struct rte_service_spec *service)
+{
+	uint32_t i;
+
+	uint64_t total_cycles = 0;
+	for (i = 0; i < rte_service_count; i++) {
+		if (!service_valid(i))
+			continue;
+		total_cycles += rte_services[i].cycles_spent;
+	}
+
+	int print_no_collect_warning = 0;
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		if (cores_state[i].collect_statistics == 0)
+			print_no_collect_warning = 1;
+	if (print_no_collect_warning)
+		fprintf(f, "Warning; cycle counts not collectd; refer to rte_service_set_stats_enable\n");
+
+	if (service) {
+		struct rte_service_spec_impl *s =
+			(struct rte_service_spec_impl *)service;
+		fprintf(f, "Service %s Summary\n", s->spec.name);
+		uint32_t reset = 0;
+		rte_service_dump_one(f, s, total_cycles, reset);
+		return 0;
+	}
+
+	fprintf(f, "Services Summary\n");
+	for (i = 0; i < rte_service_count; i++) {
+		uint32_t reset = 1;
+		rte_service_dump_one(f, &rte_services[i], total_cycles, reset);
+	}
+
+	fprintf(f, "Service Cores Summary\n");
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		if (lcore_config[i].core_role != ROLE_SERVICE)
+			continue;
+
+		uint32_t reset = 0;
+		service_dump_calls_per_lcore(f, i, reset);
+	}
+
+	return 0;
+}
diff --git a/lib/librte_eal/linuxapp/eal/Makefile b/lib/librte_eal/linuxapp/eal/Makefile
index 8651e27..e6ab6c3 100644
--- a/lib/librte_eal/linuxapp/eal/Makefile
+++ b/lib/librte_eal/linuxapp/eal/Makefile
@@ -99,6 +99,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_malloc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_elem.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_heap.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_keepalive.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_service.c
 
 # from arch dir
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_cpuflags.c
diff --git a/lib/librte_eal/linuxapp/eal/eal_thread.c b/lib/librte_eal/linuxapp/eal/eal_thread.c
index 9f88530..831ba07 100644
--- a/lib/librte_eal/linuxapp/eal/eal_thread.c
+++ b/lib/librte_eal/linuxapp/eal/eal_thread.c
@@ -184,7 +184,14 @@ eal_thread_loop(__attribute__((unused)) void *arg)
 		ret = lcore_config[lcore_id].f(fct_arg);
 		lcore_config[lcore_id].ret = ret;
 		rte_wmb();
-		lcore_config[lcore_id].state = FINISHED;
+
+		/* when a service core returns, it should go directly to WAIT
+		 * state, because the application will not lcore_wait() for it.
+		 */
+		if (lcore_config[lcore_id].core_role == ROLE_SERVICE)
+			lcore_config[lcore_id].state = WAIT;
+		else
+			lcore_config[lcore_id].state = FINISHED;
 	}
 
 	/* never reached */
diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
index a118fb1..ce5fcf0 100644
--- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
@@ -213,5 +213,27 @@ EXPERIMENTAL {
 
 	rte_eal_hotplug_add;
 	rte_eal_hotplug_remove;
+	rte_service_disable_on_lcore;
+	rte_service_dump;
+	rte_service_enable_on_lcore;
+	rte_service_get_by_id;
+	rte_service_get_by_name;
+	rte_service_get_count;
+	rte_service_get_enabled_on_lcore;
+	rte_service_is_running;
+	rte_service_lcore_add;
+	rte_service_lcore_count;
+	rte_service_lcore_del;
+	rte_service_lcore_list;
+	rte_service_lcore_reset_all;
+	rte_service_lcore_start;
+	rte_service_lcore_stop;
+	rte_service_probe_capability;
+	rte_service_register;
+	rte_service_reset;
+	rte_service_set_stats_enable;
+	rte_service_start;
+	rte_service_stop;
+	rte_service_unregister;
 
 } DPDK_17.08;
-- 
2.7.4

[PATCH v4 2/7] service cores: EAL init changes

[Harry van Haaren]

This commit shows the changes required in rte_eal_init()
to transparently launch the service threads. The threads
are launched into the service worker functions here because
after rte_eal_init() the application is not gauranteed to
call any other DPDK API.

As the registration of services happens at initialization
time, the services that require CPU time are already available
when we reach the end of rte_eal_init().

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

---

v4:
- Added #include for service cores in BSD eal.c

v2 comments:
- Include BSD implementation (Jerin)
- Move details of core-tracking into rte_service_lcore_add(Jerin)
- Given there are changes other to suggested, not using Ack
---
 lib/librte_eal/bsdapp/eal/eal.c   | 23 +++++++++++++++++++++++
 lib/librte_eal/linuxapp/eal/eal.c | 23 +++++++++++++++++++++++
 2 files changed, 46 insertions(+)

diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c
index 05f0c1f..09e3301 100644
--- a/lib/librte_eal/bsdapp/eal/eal.c
+++ b/lib/librte_eal/bsdapp/eal/eal.c
@@ -72,6 +72,7 @@
 #include <rte_common.h>
 #include <rte_version.h>
 #include <rte_atomic.h>
+#include <rte_service_private.h>
 #include <malloc_heap.h>
 
 #include "eal_private.h"
@@ -653,6 +654,17 @@ rte_eal_init(int argc, char **argv)
 	rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER);
 	rte_eal_mp_wait_lcore();
 
+	/* initialize services first so vdevs can register during bus_probe.
+	 * Ignore return value of already initialized, this means EAL parameter
+	 * -s was used to set a service-core mask.
+	 */
+	ret = rte_service_init();
+	if (ret) {
+		rte_eal_init_alert("rte_service_init() failed\n");
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	/* Probe all the buses and devices/drivers on them */
 	if (rte_bus_probe()) {
 		rte_eal_init_alert("Cannot probe devices\n");
@@ -660,6 +672,17 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 
+	/* initialize default services configuration */
+	uint32_t service_cores[RTE_MAX_LCORE];
+	int count = rte_service_lcore_list(service_cores, RTE_MAX_LCORE);
+	for (i = 0; i < count; i++)
+		rte_service_lcore_start(service_cores[i]);
+	ret = rte_service_set_default_mapping();
+	if (ret) {
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	rte_eal_mcfg_complete();
 
 	return fctret;
diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
index 7c78f2d..d63dd87 100644
--- a/lib/librte_eal/linuxapp/eal/eal.c
+++ b/lib/librte_eal/linuxapp/eal/eal.c
@@ -78,6 +78,7 @@
 #include <rte_version.h>
 #include <rte_atomic.h>
 #include <malloc_heap.h>
+#include <rte_service_private.h>
 
 #include "eal_private.h"
 #include "eal_thread.h"
@@ -932,6 +933,17 @@ rte_eal_init(int argc, char **argv)
 	rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER);
 	rte_eal_mp_wait_lcore();
 
+	/* initialize services first so vdevs can register during bus_probe.
+	 * Ignore return value of already initialized, this means EAL parameter
+	 * -s was used to set a service-core mask.
+	 */
+	ret = rte_service_init();
+	if (ret) {
+		rte_eal_init_alert("rte_service_init() failed\n");
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	/* Probe all the buses and devices/drivers on them */
 	if (rte_bus_probe()) {
 		rte_eal_init_alert("Cannot probe devices\n");
@@ -939,6 +951,17 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 
+	/* initialize default services configuration */
+	uint32_t service_cores[RTE_MAX_LCORE];
+	int count = rte_service_lcore_list(service_cores, RTE_MAX_LCORE);
+	for (i = 0; i < count; i++)
+		rte_service_lcore_start(service_cores[i]);
+	ret = rte_service_set_default_mapping();
+	if (ret) {
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	rte_eal_mcfg_complete();
 
 	return fctret;
-- 
2.7.4

[PATCH v4 3/7] service cores: coremask parsing

[Harry van Haaren]

Add logic for parsing a coremask from EAL, which allows
the application to be unaware of the cores being taken from
its coremask.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

---

v4:
- Add --help print output (Jerin)
- Fixed coremask parsing to ensure master core is ROLE_RTE (Jerin)
- Improve coremask parsing error handling, master lcore and service coremask
  overlap is failed, informing user that the core cannot be used for the
  purpose A as it is already in use for B (A or B being service/master)

v2:
- Remove printf() (Jerin)
- Remove commented code (Jerin)
- simplified core tracking, no requirement on
  #include rte_service in EAL parsing anymore.

wip: fix master lcore handling in EAL
---
 lib/librte_eal/common/eal_common_options.c | 90 +++++++++++++++++++++++++++++-
 1 file changed, 89 insertions(+), 1 deletion(-)

diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index f470195..2881884 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -61,6 +61,7 @@ const char
 eal_short_options[] =
 	"b:" /* pci-blacklist */
 	"c:" /* coremask */
+	"s:" /* service coremask */
 	"d:" /* driver */
 	"h"  /* help */
 	"l:" /* corelist */
@@ -267,6 +268,76 @@ static int xdigit2val(unsigned char c)
 }
 
 static int
+eal_parse_service_coremask(const char *coremask)
+{
+	struct rte_config *cfg = rte_eal_get_configuration();
+	int i, j, idx = 0;
+	unsigned int count = 0;
+	char c;
+	int val;
+
+	if (coremask == NULL)
+		return -1;
+	/* Remove all blank characters ahead and after .
+	 * Remove 0x/0X if exists.
+	 */
+	while (isblank(*coremask))
+		coremask++;
+	if (coremask[0] == '0' && ((coremask[1] == 'x')
+		|| (coremask[1] == 'X')))
+		coremask += 2;
+	i = strlen(coremask);
+	while ((i > 0) && isblank(coremask[i - 1]))
+		i--;
+
+	if (i == 0)
+		return -1;
+
+	for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
+		c = coremask[i];
+		if (isxdigit(c) == 0) {
+			/* invalid characters */
+			return -1;
+		}
+		val = xdigit2val(c);
+		for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE;
+				j++, idx++) {
+			if ((1 << j) & val) {
+				/* handle master lcore already parsed */
+				uint32_t lcore = idx;
+				if (master_lcore_parsed &&
+						cfg->master_lcore == lcore) {
+					RTE_LOG(ERR, EAL,
+						"Error: lcore %u is master lcore, cannot use as service core\n", idx);
+					return -1;
+				}
+
+				if (!lcore_config[idx].detected) {
+					RTE_LOG(ERR, EAL,
+						"lcore %u unavailable\n", idx);
+					return -1;
+				}
+				lcore_config[idx].core_role = ROLE_SERVICE;
+				count++;
+			}
+		}
+	}
+
+	for (; i >= 0; i--)
+		if (coremask[i] != '0')
+			return -1;
+
+	for (; idx < RTE_MAX_LCORE; idx++)
+		lcore_config[idx].core_index = -1;
+
+	if (count == 0)
+		return -1;
+
+	cfg->service_lcore_count = count;
+	return 0;
+}
+
+static int
 eal_parse_coremask(const char *coremask)
 {
 	struct rte_config *cfg = rte_eal_get_configuration();
@@ -409,6 +480,13 @@ eal_parse_master_lcore(const char *arg)
 	if (cfg->master_lcore >= RTE_MAX_LCORE)
 		return -1;
 	master_lcore_parsed = 1;
+
+	/* ensure master core is not used as service core */
+	if (lcore_config[cfg->master_lcore].core_role == ROLE_SERVICE) {
+		RTE_LOG(ERR, EAL, "Error: Master lcore is used as a service core.\n");
+		return -1;
+	}
+
 	return 0;
 }
 
@@ -826,6 +904,13 @@ eal_parse_common_option(int opt, const char *optarg,
 		}
 		core_parsed = 1;
 		break;
+	/* service coremask */
+	case 's':
+		if (eal_parse_service_coremask(optarg) < 0) {
+			RTE_LOG(ERR, EAL, "invalid service coremask\n");
+			return -1;
+		}
+		break;
 	/* size of memory */
 	case 'm':
 		conf->memory = atoi(optarg);
@@ -978,8 +1063,10 @@ eal_adjust_config(struct internal_config *internal_cfg)
 		internal_config.process_type = eal_proc_type_detect();
 
 	/* default master lcore is the first one */
-	if (!master_lcore_parsed)
+	if (!master_lcore_parsed) {
 		cfg->master_lcore = rte_get_next_lcore(-1, 0, 0);
+		lcore_config[cfg->master_lcore].core_role = ROLE_RTE;
+	}
 
 	/* if no memory amounts were requested, this will result in 0 and
 	 * will be overridden later, right after eal_hugepage_info_init() */
@@ -1052,6 +1139,7 @@ eal_common_usage(void)
 	       "                      ',' is used for single number separator.\n"
 	       "                      '( )' can be omitted for single element group,\n"
 	       "                      '@' can be omitted if cpus and lcores have the same value\n"
+	       "  -s SERVICE COREMASK Hexadecimal bitmask of cores to be used as service cores\n"
 	       "  --"OPT_MASTER_LCORE" ID   Core ID that is used as master\n"
 	       "  -n CHANNELS         Number of memory channels\n"
 	       "  -m MB               Memory to allocate (see also --"OPT_SOCKET_MEM")\n"
-- 
2.7.4

[PATCH v4 4/7] service cores: add unit tests

[Harry van Haaren]

Add a bunch of unit tests, to ensure that the service
core functions are operating as expected.

As part of these tests a dummy service is registered which
allows identifying if a service callback has been invoked
by using the CPU tick counter. This allows identifying if
functions to start and stop service lcores are actually having
effect.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

---

v4:
- Remove static slcore_id, make service-core choice dynamic (Jerin)
- Use lcore_get_next() to acquire service cores (Jerin)
- Improve unit test to ensure app can remote_launc() on a core that
  previously served as a service-core (verifies ROLE_RTE is set) (Jerin)

v2 changes;
- Rename variable to slcore_id (Jerin)
- Rename function to unregister_all() (Jerin)
- Fix typos (Jerin)
- Add unit test for get_by_name()
- Add unit tests (all suggestions by Jerin)
-- get_name()
-- Verify probe_capability API
-- Verify MT_SAFE capability (see code for details)
-- Verify rte_service_dump() API
---
 test/test/Makefile             |   2 +
 test/test/test_service_cores.c | 538 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 540 insertions(+)
 create mode 100644 test/test/test_service_cores.c

diff --git a/test/test/Makefile b/test/test/Makefile
index ee240be..61e296b 100644
--- a/test/test/Makefile
+++ b/test/test/Makefile
@@ -151,6 +151,8 @@ SRCS-y += test_interrupts.c
 SRCS-y += test_version.c
 SRCS-y += test_func_reentrancy.c
 
+SRCS-y += test_service_cores.c
+
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline.c
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_num.c
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_etheraddr.c
diff --git a/test/test/test_service_cores.c b/test/test/test_service_cores.c
new file mode 100644
index 0000000..452ea31
--- /dev/null
+++ b/test/test/test_service_cores.c
@@ -0,0 +1,538 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <rte_common.h>
+#include <rte_hexdump.h>
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_memcpy.h>
+#include <rte_cycles.h>
+
+#include <rte_service.h>
+#include <rte_service_private.h>
+
+#include "test.h"
+
+/* used as the service core ID */
+static uint32_t slcore_id;
+/* used as timestamp to detect if a service core is running */
+static uint64_t service_tick;
+/* used as a flag to check if a function was run */
+static uint32_t service_remote_launch_flag;
+
+#define SERVICE_DELAY 1
+
+#define DUMMY_SERVICE_NAME "dummy_service"
+#define MT_SAFE_SERVICE_NAME "mt_safe_service"
+
+static int
+testsuite_setup(void)
+{
+	slcore_id = rte_get_next_lcore(/* start core */ -1,
+				       /* skip master */ 1,
+				       /* wrap */ 0);
+
+	return TEST_SUCCESS;
+}
+
+static void
+testsuite_teardown(void)
+{
+	/* release service cores? */
+}
+
+static int32_t dummy_cb(void *args)
+{
+	RTE_SET_USED(args);
+	service_tick++;
+	rte_delay_ms(SERVICE_DELAY);
+	return 0;
+}
+
+
+static int32_t dummy_mt_safe_cb(void *args)
+{
+	/* Atomic checks to ensure MT safe services allow > 1 thread to
+	 * concurrently run the callback. The concept is as follows;
+	 * 1) if lock is available, take the lock then delay
+	 * 2) if first lock is taken, and a thread arrives in the CB, we know
+	 *    that 2 threads are running the callback at the same time: MT safe
+	 */
+	uint32_t *test_params = args;
+	uint32_t *atomic_lock = &test_params[0];
+	uint32_t *pass_test = &test_params[1];
+	int lock_taken = rte_atomic32_cmpset(atomic_lock, 0, 1);
+	if (lock_taken) {
+		/* delay with the lock held */
+		rte_delay_ms(250);
+		rte_atomic32_clear((rte_atomic32_t *)atomic_lock);
+	} else {
+		/* 2nd thread will fail to take lock, so set pass flag */
+		*pass_test = 1;
+	}
+
+	return 0;
+}
+
+/* unregister all services */
+static int
+unregister_all(void)
+{
+	uint32_t i;
+	struct rte_service_spec *dead = (struct rte_service_spec *)0xdead;
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_unregister(0),
+			"Unregistered NULL pointer");
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_unregister(dead),
+			"Unregistered invalid pointer");
+
+	uint32_t c = rte_service_get_count();
+	for (i = 0; i < c; i++) {
+		struct rte_service_spec *s = rte_service_get_by_id(i);
+		TEST_ASSERT_EQUAL(0, rte_service_unregister(s),
+				"Error unregistering a valid service");
+	}
+
+	rte_service_lcore_reset_all();
+
+	return TEST_SUCCESS;
+}
+
+/* register a single dummy service */
+static int
+dummy_register(void)
+{
+	/* make sure there are no remains from previous tests */
+	unregister_all();
+
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid callback");
+	service.callback = dummy_cb;
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid name");
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Failed to register valid service");
+
+	return TEST_SUCCESS;
+}
+
+/* verify get_by_name() service lookup */
+static int
+service_get_by_name(void)
+{
+	unregister_all();
+
+	/* ensure with no services registered returns NULL */
+	TEST_ASSERT_EQUAL(0, rte_service_get_by_name(DUMMY_SERVICE_NAME),
+			"Service get by name should return NULL");
+
+	/* register service */
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid callback");
+	service.callback = dummy_cb;
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid name");
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Failed to register valid service");
+
+	/* ensure with dummy services registered returns same ptr as ID */
+	struct rte_service_spec *s_by_id = rte_service_get_by_id(0);
+	TEST_ASSERT_EQUAL(s_by_id, rte_service_get_by_name(DUMMY_SERVICE_NAME),
+			"Service get_by_name should equal get_by_id()");
+
+	unregister_all();
+
+	/* ensure after unregister, get_by_name returns NULL */
+	TEST_ASSERT_EQUAL(0, rte_service_get_by_name(DUMMY_SERVICE_NAME),
+			"get by name should return NULL after unregister");
+
+	return TEST_SUCCESS;
+}
+
+/* verify probe of capabilities */
+static int
+service_probe_capability(void)
+{
+	unregister_all();
+
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	service.callback = dummy_cb;
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+	service.capabilities |= RTE_SERVICE_CAP_MT_SAFE;
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Register of MT SAFE service failed");
+
+	/* verify flag is enabled */
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+	int32_t mt = rte_service_probe_capability(s, RTE_SERVICE_CAP_MT_SAFE);
+	TEST_ASSERT_EQUAL(1, mt, "MT SAFE capability flag not set.");
+
+
+	unregister_all();
+
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	service.callback = dummy_cb;
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Register of non-MT safe service failed");
+
+	/* verify flag is enabled */
+	s = rte_service_get_by_id(0);
+	mt = rte_service_probe_capability(s, RTE_SERVICE_CAP_MT_SAFE);
+	TEST_ASSERT_EQUAL(0, mt, "MT SAFE cap flag set on non MT SAFE service");
+
+	return unregister_all();
+}
+
+/* verify the service name */
+static int
+service_name(void)
+{
+	struct rte_service_spec *service = rte_service_get_by_id(0);
+
+	int equal = strcmp(service->name, DUMMY_SERVICE_NAME);
+	TEST_ASSERT_EQUAL(0, equal, "Error: Service name not correct");
+
+	return unregister_all();
+}
+
+/* verify service dump */
+static int
+service_dump(void)
+{
+	struct rte_service_spec *service = rte_service_get_by_id(0);
+	rte_service_set_stats_enable(1);
+	rte_service_dump(stdout, service);
+	return unregister_all();
+}
+
+/* start and stop a service */
+static int
+service_start_stop(void)
+{
+	struct rte_service_spec *service = rte_service_get_by_id(0);
+
+	/* is_running() returns if service is running and slcore is mapped */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Service core add did not return zero");
+	int ret = rte_service_enable_on_lcore(service, slcore_id);
+	TEST_ASSERT_EQUAL(0, ret,
+			"Enabling service core, expected 0 got %d", ret);
+
+	TEST_ASSERT_EQUAL(0, rte_service_is_running(service),
+			"Error: Service should be stopped");
+
+	TEST_ASSERT_EQUAL(0, rte_service_stop(service),
+			"Error: Service stopped returned non-zero");
+
+	TEST_ASSERT_EQUAL(0, rte_service_is_running(service),
+			"Error: Service is running - should be stopped");
+
+	TEST_ASSERT_EQUAL(0, rte_service_start(service),
+			"Error: Service start returned non-zero");
+
+	TEST_ASSERT_EQUAL(1, rte_service_is_running(service),
+			"Error: Service is not running");
+
+	return unregister_all();
+}
+
+
+static int
+service_remote_launch_func(void *arg)
+{
+	RTE_SET_USED(arg);
+	service_remote_launch_flag = 1;
+	return 0;
+}
+
+/* enable and disable a lcore for a service */
+static int
+service_lcore_en_dis_able(void)
+{
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+
+	/* expected failure cases */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_enable_on_lcore(s, 100000),
+			"Enable on invalid core did not fail");
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_disable_on_lcore(s, 100000),
+			"Disable on invalid core did not fail");
+
+	/* add service core to allow enabling */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Add service core failed when not in use before");
+
+	/* valid enable */
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, slcore_id),
+			"Enabling valid service and core failed");
+	TEST_ASSERT_EQUAL(1, rte_service_get_enabled_on_lcore(s, slcore_id),
+			"Enabled core returned not-enabled");
+
+	/* valid disable */
+	TEST_ASSERT_EQUAL(0, rte_service_disable_on_lcore(s, slcore_id),
+			"Disabling valid service and lcore failed");
+	TEST_ASSERT_EQUAL(0, rte_service_get_enabled_on_lcore(s, slcore_id),
+			"Disabled core returned enabled");
+
+	/* call remote_launch to verify that app can launch ex-service lcore */
+	service_remote_launch_flag = 0;
+	int ret = rte_eal_remote_launch(service_remote_launch_func, NULL,
+					slcore_id);
+	TEST_ASSERT_EQUAL(0, ret, "Ex-service core remote launch failed.");
+	rte_eal_mp_wait_lcore();
+	TEST_ASSERT_EQUAL(1, service_remote_launch_flag,
+			"Ex-service core function call had no effect.");
+
+	return unregister_all();
+}
+
+static int
+service_lcore_running_check(void)
+{
+	uint64_t tick = service_tick;
+	rte_delay_ms(SERVICE_DELAY * 10);
+	/* if (tick != service_tick) we know the lcore as polled the service */
+	return tick != service_tick;
+}
+
+static int
+service_lcore_add_del(void)
+{
+	/* check initial count */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_count(),
+			"Service lcore count has value before adding a lcore");
+
+	/* check service lcore add */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Add service core failed when not in use before");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_add(slcore_id),
+			"Add service core failed to refuse in-use lcore");
+
+	/* check count */
+	TEST_ASSERT_EQUAL(1, rte_service_lcore_count(),
+			"Service core count not equal to one");
+
+	/* retrieve core list, checking lcore ids */
+	const uint32_t size = 4;
+	uint32_t service_core_ids[size];
+	int32_t n = rte_service_lcore_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(1, n, "Service core list return should equal 1");
+	TEST_ASSERT_EQUAL(slcore_id, service_core_ids[0],
+				"Service core list lcore must equal slcore_id");
+
+	/* recheck count, add more cores, and check count */
+	TEST_ASSERT_EQUAL(1, rte_service_lcore_count(),
+			"Service core count not equal to one");
+	uint32_t slcore_1 = rte_get_next_lcore(/* start core */ -1,
+					       /* skip master */ 1,
+					       /* wrap */ 0);
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_1),
+			"Service core add did not return zero");
+	uint32_t slcore_2 = rte_get_next_lcore(/* start core */ slcore_1,
+					       /* skip master */ 1,
+					       /* wrap */ 0);
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_2),
+			"Service core add did not return zero");
+
+	uint32_t count = rte_service_lcore_count();
+	const uint32_t cores_at_this_point = 3;
+	TEST_ASSERT_EQUAL(cores_at_this_point, count,
+			"Service core count %d, expected %d", count,
+			cores_at_this_point);
+
+	/* check longer service core list */
+	n = rte_service_lcore_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(3, n, "Service core list return should equal 3");
+	TEST_ASSERT_EQUAL(slcore_id, service_core_ids[0],
+				"Service core list[0] lcore must equal 1");
+	TEST_ASSERT_EQUAL(slcore_1, service_core_ids[1],
+				"Service core list[1] lcore must equal 2");
+	TEST_ASSERT_EQUAL(slcore_2, service_core_ids[2],
+				"Service core list[2] lcore must equal 3");
+
+	/* recheck count, remove lcores, check remaining lcore_id is correct */
+	TEST_ASSERT_EQUAL(3, rte_service_lcore_count(),
+			"Service core count not equal to three");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_del(slcore_1),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_del(slcore_2),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(1, rte_service_lcore_count(),
+			"Service core count not equal to one");
+	n = rte_service_lcore_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(1, n, "Service core list return should equal one");
+	TEST_ASSERT_EQUAL(slcore_id, service_core_ids[0],
+				"Service core list[0] lcore must equal %d",
+				slcore_id);
+
+	return unregister_all();
+}
+
+static int
+service_mt_safe_poll(void)
+{
+	unregister_all();
+
+	/* add next 2 cores */
+	uint32_t slcore_1 = rte_get_next_lcore(/* start core */ -1,
+					       /* skip master */ 1,
+					       /* wrap */ 0);
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_1),
+			"mt safe lcore add fail");
+	uint32_t slcore_2 = rte_get_next_lcore(/* start core */ slcore_1,
+					       /* skip master */ 1,
+					       /* wrap */ 0);
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_2),
+			"mt safe lcore add fail");
+
+	/* Use atomic locks to verify that two threads are in the same function
+	 * at the same time. These are passed to the unit tests through the
+	 * callback userdata parameter
+	 */
+	uint32_t test_params[2];
+	memset(test_params, 0, sizeof(uint32_t) * 2);
+
+	/* register MT safe service. */
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	service.callback = dummy_mt_safe_cb;
+	service.callback_userdata = test_params;
+	snprintf(service.name, sizeof(service.name), MT_SAFE_SERVICE_NAME);
+	service.capabilities |= RTE_SERVICE_CAP_MT_SAFE;
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Register of MT SAFE service failed");
+
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+	TEST_ASSERT_EQUAL(0, rte_service_start(s),
+			"Starting valid service failed");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, slcore_1),
+			"Failed to enable lcore 1 on mt safe service");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, slcore_2),
+			"Failed to enable lcore 2 on mt safe service");
+	rte_service_lcore_start(slcore_1);
+	rte_service_lcore_start(slcore_2);
+
+	/* wait for the worker threads to run */
+	rte_delay_ms(500);
+	rte_service_lcore_stop(slcore_1);
+	rte_service_lcore_stop(slcore_2);
+
+	TEST_ASSERT_EQUAL(1, test_params[1],
+			"MT Safe service not run by two cores concurrently");
+
+	TEST_ASSERT_EQUAL(0, rte_service_stop(s),
+			"Failed to stop MT Safe service");
+
+	return unregister_all();
+}
+
+/* start and stop a service core - ensuring it goes back to sleep */
+static int
+service_lcore_start_stop(void)
+{
+	/* start service core and service, create mapping so tick() runs */
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+	TEST_ASSERT_EQUAL(0, rte_service_start(s),
+			"Starting valid service failed");
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_enable_on_lcore(s, slcore_id),
+			"Enabling valid service on non-service core must fail");
+
+	/* core start */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_lcore_start(slcore_id),
+			"Service core start without add should return EINVAL");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, slcore_id),
+			"Enabling valid service on valid core failed");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_start(slcore_id),
+			"Service core start after add failed");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_start(slcore_id),
+			"Service core expected as running but was stopped");
+
+	/* ensures core really is running the service function */
+	TEST_ASSERT_EQUAL(1, service_lcore_running_check(),
+			"Service core expected to poll service but it didn't");
+
+	/* core stop */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_lcore_stop(100000),
+			"Invalid Service core stop should return -EINVAL");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_stop(slcore_id),
+			"Service core stop expected to return 0");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_stop(slcore_id),
+			"Already stopped service core should return -EALREADY");
+
+	/* ensure service is not longer running */
+	TEST_ASSERT_EQUAL(0, service_lcore_running_check(),
+			"Service core expected to poll service but it didn't");
+
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_del(slcore_id),
+			"Service core del did not return zero");
+
+	return unregister_all();
+}
+
+static struct unit_test_suite service_tests  = {
+	.suite_name = "service core test suite",
+	.setup = testsuite_setup,
+	.teardown = testsuite_teardown,
+	.unit_test_cases = {
+		TEST_CASE_ST(dummy_register, NULL, unregister_all),
+		TEST_CASE_ST(dummy_register, NULL, service_name),
+		TEST_CASE_ST(dummy_register, NULL, service_get_by_name),
+		TEST_CASE_ST(dummy_register, NULL, service_dump),
+		TEST_CASE_ST(dummy_register, NULL, service_probe_capability),
+		TEST_CASE_ST(dummy_register, NULL, service_start_stop),
+		TEST_CASE_ST(dummy_register, NULL, service_lcore_add_del),
+		TEST_CASE_ST(dummy_register, NULL, service_lcore_start_stop),
+		TEST_CASE_ST(dummy_register, NULL, service_lcore_en_dis_able),
+		TEST_CASE_ST(dummy_register, NULL, service_mt_safe_poll),
+		TEST_CASES_END() /**< NULL terminate unit test array */
+	}
+};
+
+static int
+test_service_common(void)
+{
+	return unit_test_suite_runner(&service_tests);
+}
+
+REGISTER_TEST_COMMAND(service_autotest, test_service_common);
-- 
2.7.4

[PATCH v4 5/7] event/sw: enable SW PMD with service capability

[Harry van Haaren]

This commit shows how easy it is to enable a specific
DPDK component with a service callback, in order to get
CPU cycles for it.

The beauty of this method is that the service is unaware
of how much CPU time it is getting - the application can
decide how to split and slice cores and map them to the
registered services.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

---

v4:
- Include Acked by

v2:
- Remove #include <rte_cycles.h> (Jerin)
- Remove development prints (Jerin)
- Track service name in PMD
- Print warning if service does not have an lcore mapped (Jerin)
---
 drivers/event/sw/sw_evdev.c | 32 ++++++++++++++++++++++++++++++++
 drivers/event/sw/sw_evdev.h |  3 +++
 2 files changed, 35 insertions(+)

diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
index fe2a61e..baab376 100644
--- a/drivers/event/sw/sw_evdev.c
+++ b/drivers/event/sw/sw_evdev.c
@@ -38,6 +38,7 @@
 #include <rte_kvargs.h>
 #include <rte_ring.h>
 #include <rte_errno.h>
+#include <rte_service_private.h>
 
 #include "sw_evdev.h"
 #include "iq_ring.h"
@@ -597,6 +598,13 @@ sw_start(struct rte_eventdev *dev)
 {
 	unsigned int i, j;
 	struct sw_evdev *sw = sw_pmd_priv(dev);
+
+	/* check a service core is mapped to this service */
+	struct rte_service_spec *s = rte_service_get_by_name(sw->service_name);
+	if (!rte_service_is_running(s))
+		SW_LOG_ERR("Warning: No Service core enabled on service %s\n",
+				s->name);
+
 	/* check all ports are set up */
 	for (i = 0; i < sw->port_count; i++)
 		if (sw->ports[i].rx_worker_ring == NULL) {
@@ -699,6 +707,14 @@ set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque)
 	return 0;
 }
 
+
+static int32_t sw_sched_service_func(void *args)
+{
+	struct rte_eventdev *dev = args;
+	sw_event_schedule(dev);
+	return 0;
+}
+
 static int
 sw_probe(struct rte_vdev_device *vdev)
 {
@@ -810,6 +826,22 @@ sw_probe(struct rte_vdev_device *vdev)
 	sw->credit_update_quanta = credit_quanta;
 	sw->sched_quanta = sched_quanta;
 
+	/* register service with EAL */
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	snprintf(service.name, sizeof(service.name), "%s_service", name);
+	snprintf(sw->service_name, sizeof(sw->service_name), "%s_service",
+			name);
+	service.socket_id = socket_id;
+	service.callback = sw_sched_service_func;
+	service.callback_userdata = (void *)dev;
+
+	int32_t ret = rte_service_register(&service);
+	if (ret) {
+		SW_LOG_ERR("service register() failed");
+		return -ENOEXEC;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/event/sw/sw_evdev.h b/drivers/event/sw/sw_evdev.h
index 0d7f94f..3e83823 100644
--- a/drivers/event/sw/sw_evdev.h
+++ b/drivers/event/sw/sw_evdev.h
@@ -59,6 +59,7 @@
 
 #define EVENTDEV_NAME_SW_PMD event_sw
 #define SW_PMD_NAME RTE_STR(event_sw)
+#define SW_PMD_NAME_MAX 64
 
 #define SW_SCHED_TYPE_DIRECT (RTE_SCHED_TYPE_PARALLEL + 1)
 
@@ -276,6 +277,8 @@ struct sw_evdev {
 	/* store num stats and offset of the stats for each queue */
 	uint16_t xstats_count_per_qid[RTE_EVENT_MAX_QUEUES_PER_DEV];
 	uint16_t xstats_offset_for_qid[RTE_EVENT_MAX_QUEUES_PER_DEV];
+
+	char service_name[SW_PMD_NAME_MAX];
 };
 
 static inline struct sw_evdev *
-- 
2.7.4

[PATCH v4 6/7] doc: add service cores to doc and release notes

[Harry van Haaren]

Add a section describing the fundamental concepts behind service
cores. Where service cores originate from, and how to enable services.
The release notes for 17.08 are updated, with an introductory paragraph
on the service cores concept. Finally the Eventdev SW PMD documentation
is amended to reflect that it can be run as a service.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

---

v4:
- Add ack from ML

I would like to enable the service-cores in the eventdev_pipeline
sample app, to showcase the power of the service-core abstraction.
There is some remaining work TODO, in order to genericise the sample
app for both HW and SW PMDs, and during that rework the service-cores
can be added too.

The sample app will make a good showcase for docs, and make it much
easier to understand.
---
 doc/guides/eventdevs/sw.rst             |  4 +-
 doc/guides/prog_guide/index.rst         |  1 +
 doc/guides/prog_guide/service_cores.rst | 81 +++++++++++++++++++++++++++++++++
 doc/guides/rel_notes/release_17_08.rst  |  8 ++++
 4 files changed, 93 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/prog_guide/service_cores.rst

diff --git a/doc/guides/eventdevs/sw.rst b/doc/guides/eventdevs/sw.rst
index fb63c84..a3e6624 100644
--- a/doc/guides/eventdevs/sw.rst
+++ b/doc/guides/eventdevs/sw.rst
@@ -32,7 +32,9 @@ Software Eventdev Poll Mode Driver
 
 The software eventdev is an implementation of the eventdev API, that provides a
 wide range of the eventdev features. The eventdev relies on a CPU core to
-perform event scheduling.
+perform event scheduling. This PMD can use the service core library to run the
+scheduling function, allowing an application to utilize the power of service
+cores to multiplex other work on the same core if required.
 
 
 Features
diff --git a/doc/guides/prog_guide/index.rst b/doc/guides/prog_guide/index.rst
index ef5a02a..231622a 100644
--- a/doc/guides/prog_guide/index.rst
+++ b/doc/guides/prog_guide/index.rst
@@ -38,6 +38,7 @@ Programmer's Guide
     intro
     overview
     env_abstraction_layer
+    service_cores
     ring_lib
     mempool_lib
     mbuf_lib
diff --git a/doc/guides/prog_guide/service_cores.rst b/doc/guides/prog_guide/service_cores.rst
new file mode 100644
index 0000000..3a029ba
--- /dev/null
+++ b/doc/guides/prog_guide/service_cores.rst
@@ -0,0 +1,81 @@
+..  BSD LICENSE
+    Copyright(c) 2017 Intel Corporation. All rights reserved.
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions
+    are met:
+
+    * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+    * Neither the name of Intel Corporation nor the names of its
+    contributors may be used to endorse or promote products derived
+    from this software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Service Cores
+=============
+
+DPDK has a concept known as service cores, which enables a dynamic way of
+performing work on DPDK lcores. Service core support is built into the EAL, and
+an API is provided to optionally allow applications to control how the service
+cores are used at runtime.
+
+The service cores concept is built up out of services (components of DPDK that
+require CPU cycles to operate) and service cores (DPDK lcores, tasked with
+running services). The power of the service core concept is that the mapping
+between service cores and services can be configured to abstract away the
+difference between platforms and environments.
+
+For example, the Eventdev has hardware and software PMDs. Of these the software
+PMD requires an lcore to perform the scheduling operations, while the hardware
+PMD does not. With service cores, the application would not directly notice
+that the scheduling is done in software.
+
+For detailed information about the service core API, please refer to the docs.
+
+Service Core Initialization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are two methods to having service cores in a DPDK application, either by
+using the service coremask, or by dynamically adding cores using the API.
+The simpler of the two is to pass the `-s` coremask argument to EAL, which will
+take any cores available in the main DPDK coremask, an if the bits are also set
+in the service coremask the cores become service-cores instead of DPDK
+application lcores.
+
+Enabling Services on Cores
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Each registered service can be individually mapped to a service core, or set of
+service cores. Enabling a service on a particular core means that the lcore in
+question will run the service. Disabling that core on the service stops the
+lcore in question from running the service.
+
+Using this method, it is possible to assign specific workloads to each
+service core, and map N workloads to M number of service cores. Each service
+lcore loops over the services that are enabled for that core, and invokes the
+function to run the service.
+
+Service Core Statistics
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The service core library is capable of collecting runtime statistics like number
+of calls to a specific service, and number of cycles used by the service. The
+cycle count collection is dynamically configurable, allowing any application to
+profile the services running on the system at any time.
diff --git a/doc/guides/rel_notes/release_17_08.rst b/doc/guides/rel_notes/release_17_08.rst
index 76bb179..30acb30 100644
--- a/doc/guides/rel_notes/release_17_08.rst
+++ b/doc/guides/rel_notes/release_17_08.rst
@@ -51,6 +51,14 @@ New Features
     RAW pattern items with QUEUE actions. There are four
     type of filter support for this feature on igb.
 
+* **Added Service Core functionality.**
+
+  The service core functionality added to EAL allows DPDK to run services such
+  as SW PMDs on lcores without the application manually running them. The
+  service core infrastructure allows flexibility of running multiple services
+  on the same service lcore, and provides the application with powerful APIs to
+  configure the mapping from service lcores to services.
+
 * **Added Generic Flow API support to enic.**
 
   Flow API support for outer Ethernet, VLAN, IPv4, IPv6, UDP, TCP, SCTP, VxLAN
-- 
2.7.4

[PATCH v4 7/7] maintainers: claim service cores

[Harry van Haaren]

Sign-up to be the maintainer of public header files and
implementation of the service-cores infrastructure.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

---

v4:
- fix rte_service.c filename
- Include service_cores.rst file
---
 MAINTAINERS | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/MAINTAINERS b/MAINTAINERS
index b4424ea..1cf6bf6 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -134,6 +134,13 @@ F: test/test/test_mp_secondary.c
 F: examples/multi_process/
 F: doc/guides/sample_app_ug/multi_process.rst
 
+Service Cores
+M: Harry van Haaren <harry.van.haaren@intel.com>
+F: doc/guides/prog_guide/service_cores.rst
+F: lib/librte_eal/common/include/rte_service.h
+F: lib/librte_eal/common/include/rte_service_private.h
+F: lib/librte_eal/common/rte_service.c
+
 ARM v7
 M: Jan Viktorin <viktorin@rehivetech.com>
 M: Jianbo Liu <jianbo.liu@linaro.org>
-- 
2.7.4

Re: [PATCH v4 0/7] service cores: cover letter

[Thomas Monjalon]

07/07/2017 18:41, Harry van Haaren:
> v4:
> - Range of fixes as suggested by Jerin
> - Improved unit tests, ensuring ex-service cores become available to app
> - Added functions to EXPERIMENTAL tag in .map files (Thomas)
> - Added @warning experimental notes to Doxygen API documentation (Thomas)
> - Various smaller fixes / cleanups
> - See commit notes for details

I feel we need to wait approval from some reviewers, at least from Jerin
who was very active on this series.
It means it misses the first release candidate, and it is a problem
because this feature is updating the core.

Naive question: do we really want it in 17.08?

Re: [PATCH v4 0/7] service cores: cover letter

[Van Haaren, Harry]

> From: Thomas Monjalon [mailto:thomas@monjalon.net]
> Sent: Sunday, July 9, 2017 11:09 PM
> To: Van Haaren, Harry <harry.van.haaren@intel.com>
> Cc: dev@dpdk.org; jerin.jacob@caviumnetworks.com; Wiles, Keith <keith.wiles@intel.com>;
> Richardson, Bruce <bruce.richardson@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v4 0/7] service cores: cover letter
> 
> 07/07/2017 18:41, Harry van Haaren:
> > v4:
> > - Range of fixes as suggested by Jerin
> > - Improved unit tests, ensuring ex-service cores become available to app
> > - Added functions to EXPERIMENTAL tag in .map files (Thomas)
> > - Added @warning experimental notes to Doxygen API documentation (Thomas)
> > - Various smaller fixes / cleanups
> > - See commit notes for details
> 
> I feel we need to wait approval from some reviewers, at least from Jerin
> who was very active on this series.

Agreed that more reviewing eyes would be good.


> It means it misses the first release candidate, and it is a problem
> because this feature is updating the core.

Correct service cores does update the core, however a review of the code-paths
taken if the service cores -s switch is not used shows that it will have almost
no impact on the EAL code path.


> Naive question: do we really want it in 17.08?

I think merging in 17.08 allow us to progress on integrating services and DPDK, so I see value in including in this release.

Re: [PATCH v4 0/7] service cores: cover letter

[Jerin Jacob]

-----Original Message-----
> Date: Mon, 10 Jul 2017 08:18:33 +0000
> From: "Van Haaren, Harry" <harry.van.haaren@intel.com>
> To: Thomas Monjalon <thomas@monjalon.net>
> CC: "dev@dpdk.org" <dev@dpdk.org>, "jerin.jacob@caviumnetworks.com"
>  <jerin.jacob@caviumnetworks.com>, "Wiles, Keith" <keith.wiles@intel.com>,
>  "Richardson, Bruce" <bruce.richardson@intel.com>
> Subject: RE: [dpdk-dev] [PATCH v4 0/7] service cores: cover letter
> 
> > From: Thomas Monjalon [mailto:thomas@monjalon.net]
> > Sent: Sunday, July 9, 2017 11:09 PM
> > To: Van Haaren, Harry <harry.van.haaren@intel.com>
> > Cc: dev@dpdk.org; jerin.jacob@caviumnetworks.com; Wiles, Keith <keith.wiles@intel.com>;
> > Richardson, Bruce <bruce.richardson@intel.com>
> > Subject: Re: [dpdk-dev] [PATCH v4 0/7] service cores: cover letter
> > 
> > 07/07/2017 18:41, Harry van Haaren:
> > > v4:
> > > - Range of fixes as suggested by Jerin
> > > - Improved unit tests, ensuring ex-service cores become available to app
> > > - Added functions to EXPERIMENTAL tag in .map files (Thomas)
> > > - Added @warning experimental notes to Doxygen API documentation (Thomas)
> > > - Various smaller fixes / cleanups
> > > - See commit notes for details
> > 
> > I feel we need to wait approval from some reviewers, at least from Jerin
> > who was very active on this series.
> 
> Agreed that more reviewing eyes would be good.
> 
> 
> > It means it misses the first release candidate, and it is a problem
> > because this feature is updating the core.
> 
> Correct service cores does update the core, however a review of the code-paths
> taken if the service cores -s switch is not used shows that it will have almost
> no impact on the EAL code path.
> 
> 
> > Naive question: do we really want it in 17.08?
> 
> I think merging in 17.08 allow us to progress on integrating services and DPDK, so I see value in including in this release.

+1

Re: [PATCH v4 2/7] service cores: EAL init changes

[Jerin Jacob]

-----Original Message-----
> Date: Fri, 7 Jul 2017 17:41:02 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: thomas@monjalon.net, jerin.jacob@caviumnetworks.com,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH v4 2/7] service cores: EAL init changes
> X-Mailer: git-send-email 2.7.4
> 
> This commit shows the changes required in rte_eal_init()
> to transparently launch the service threads. The threads
> are launched into the service worker functions here because
> after rte_eal_init() the application is not gauranteed to
> call any other DPDK API.
> 
> As the registration of services happens at initialization
> time, the services that require CPU time are already available
> when we reach the end of rte_eal_init().
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> 
> ---
> 
> v4:
> - Added #include for service cores in BSD eal.c
> 
> v2 comments:
> - Include BSD implementation (Jerin)
> - Move details of core-tracking into rte_service_lcore_add(Jerin)
> - Given there are changes other to suggested, not using Ack
> ---
>  lib/librte_eal/bsdapp/eal/eal.c   | 23 +++++++++++++++++++++++
>  lib/librte_eal/linuxapp/eal/eal.c | 23 +++++++++++++++++++++++
>  2 files changed, 46 insertions(+)
> 
> diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c
> index 05f0c1f..09e3301 100644
> --- a/lib/librte_eal/bsdapp/eal/eal.c
> +++ b/lib/librte_eal/bsdapp/eal/eal.c
> @@ -72,6 +72,7 @@
>  #include <rte_common.h>
>  #include <rte_version.h>
>  #include <rte_atomic.h>
> +#include <rte_service_private.h>
>  #include <malloc_heap.h>
>  
>  #include "eal_private.h"
> @@ -653,6 +654,17 @@ rte_eal_init(int argc, char **argv)
>  	rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER);
>  	rte_eal_mp_wait_lcore();
>  
> +	/* initialize services first so vdevs can register during bus_probe.
> +	 * Ignore return value of already initialized, this means EAL parameter
> +	 * -s was used to set a service-core mask.
> +	 */
> +	ret = rte_service_init();
> +	if (ret) {
> +		rte_eal_init_alert("rte_service_init() failed\n");
> +		rte_errno = ENOEXEC;
> +		return -1;
> +	}
> +
>  	/* Probe all the buses and devices/drivers on them */
>  	if (rte_bus_probe()) {
>  		rte_eal_init_alert("Cannot probe devices\n");
> @@ -660,6 +672,17 @@ rte_eal_init(int argc, char **argv)
>  		return -1;
>  	}
>  
> +	/* initialize default services configuration */
> +	uint32_t service_cores[RTE_MAX_LCORE];
> +	int count = rte_service_lcore_list(service_cores, RTE_MAX_LCORE);
> +	for (i = 0; i < count; i++)
> +		rte_service_lcore_start(service_cores[i]);
> +	ret = rte_service_set_default_mapping();
> +	if (ret) {
> +		rte_errno = ENOEXEC;
> +		return -1;
> +	}

How about moving, rte_service_lcore_start() inside
rte_service_set_default_mapping() so that rte_eal_init() level change will be
less in linuxapp and bsdapp?(and both changes are tightly coupled too).

You could change the function name to rte_service_enable_default_mapping()
or something like that to include rte_service_lcore_start() start change.

With that change:
Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

Re: [PATCH v4 7/7] maintainers: claim service cores

[Jerin Jacob]

-----Original Message-----
> Date: Fri, 7 Jul 2017 17:41:07 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: thomas@monjalon.net, jerin.jacob@caviumnetworks.com,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH v4 7/7] maintainers: claim service cores
> X-Mailer: git-send-email 2.7.4
> 
> Sign-up to be the maintainer of public header files and
> implementation of the service-cores infrastructure.
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>


Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

> 
> ---
> 
> v4:
> - fix rte_service.c filename
> - Include service_cores.rst file
> ---
>  MAINTAINERS | 7 +++++++
>  1 file changed, 7 insertions(+)
> 
> diff --git a/MAINTAINERS b/MAINTAINERS
> index b4424ea..1cf6bf6 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -134,6 +134,13 @@ F: test/test/test_mp_secondary.c
>  F: examples/multi_process/
>  F: doc/guides/sample_app_ug/multi_process.rst
>  
> +Service Cores
> +M: Harry van Haaren <harry.van.haaren@intel.com>
> +F: doc/guides/prog_guide/service_cores.rst
> +F: lib/librte_eal/common/include/rte_service.h
> +F: lib/librte_eal/common/include/rte_service_private.h
> +F: lib/librte_eal/common/rte_service.c
> +
>  ARM v7
>  M: Jan Viktorin <viktorin@rehivetech.com>
>  M: Jianbo Liu <jianbo.liu@linaro.org>
> -- 
> 2.7.4
> 

Re: [PATCH v4 4/7] service cores: add unit tests

[Jerin Jacob]

-----Original Message-----
> Date: Fri, 7 Jul 2017 17:41:04 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: thomas@monjalon.net, jerin.jacob@caviumnetworks.com,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH v4 4/7] service cores: add unit tests
> X-Mailer: git-send-email 2.7.4
> 
> Add a bunch of unit tests, to ensure that the service
> core functions are operating as expected.
> 
> As part of these tests a dummy service is registered which
> allows identifying if a service callback has been invoked
> by using the CPU tick counter. This allows identifying if
> functions to start and stop service lcores are actually having
> effect.
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> 
> ---
> 
> v4:
> - Remove static slcore_id, make service-core choice dynamic (Jerin)
> - Use lcore_get_next() to acquire service cores (Jerin)
> - Improve unit test to ensure app can remote_launc() on a core that
>   previously served as a service-core (verifies ROLE_RTE is set) (Jerin)

IMO, it has the unit testcases for every API now.

Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

Re: [PATCH v4 1/7] service cores: header and implementation

[Jerin Jacob]

-----Original Message-----
> Date: Fri, 7 Jul 2017 17:41:01 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: thomas@monjalon.net, jerin.jacob@caviumnetworks.com,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH v4 1/7] service cores: header and implementation
> X-Mailer: git-send-email 2.7.4
> 
> Add header files, update .map files with new service
> functions, and add the service header to the doxygen
> for building.
> 
> This service header API allows DPDK to use services as
> a concept of something that requires CPU cycles. An example
> is a PMD that runs in software to schedule events, where a
> hardware version exists that does not require a CPU.
> 
> The code presented here is based on an initial RFC:
> http://dpdk.org/ml/archives/dev/2017-May/065207.html
> This was then reworked, and RFC v2 with the changes posted:
> http://dpdk.org/ml/archives/dev/2017-June/067194.html
> 
> This is the fourth iteration of the service core concept,
> with 2 RFCs and this being v2 of the implementation.

Remove above info from the git commit.

> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> 
> ---
>  doc/api/doxy-api-index.md                          |   1 +
>  lib/librte_eal/bsdapp/eal/Makefile                 |   1 +
>  lib/librte_eal/bsdapp/eal/rte_eal_version.map      |  22 +
>  lib/librte_eal/common/Makefile                     |   1 +
>  lib/librte_eal/common/eal_common_lcore.c           |   1 +
>  lib/librte_eal/common/include/rte_eal.h            |   4 +
>  lib/librte_eal/common/include/rte_lcore.h          |   3 +-
>  lib/librte_eal/common/include/rte_service.h        | 383 ++++++++++++
>  .../common/include/rte_service_private.h           | 140 +++++
>  lib/librte_eal/common/rte_service.c                | 687 +++++++++++++++++++++
>  lib/librte_eal/linuxapp/eal/Makefile               |   1 +
>  lib/librte_eal/linuxapp/eal/eal_thread.c           |   9 +-
>  lib/librte_eal/linuxapp/eal/rte_eal_version.map    |  22 +
>  13 files changed, 1273 insertions(+), 2 deletions(-)
>  create mode 100644 lib/librte_eal/common/include/rte_service.h
>  create mode 100644 lib/librte_eal/common/include/rte_service_private.h
>  create mode 100644 lib/librte_eal/common/rte_service.c
> 
> diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
> index 3b83288..e2abdf4 100644
> --- a/doc/api/doxy-api-index.md
> +++ b/doc/api/doxy-api-index.md
> @@ -159,6 +159,7 @@ There are many libraries, so their headers may be grouped by topics:
>    [common]             (@ref rte_common.h),
>    [ABI compat]         (@ref rte_compat.h),
>    [keepalive]          (@ref rte_keepalive.h),
> +  [service cores]      (@ref rte_service.h),
>    [device metrics]     (@ref rte_metrics.h),
>    [bitrate statistics] (@ref rte_bitrate.h),
>    [latency statistics] (@ref rte_latencystats.h),


Fix the below mentioned documentation warning.

+/export/dpdk.org/lib/librte_eal/common/include/rte_service.h:338:
warning: argument 'enabled' of command @param is not found in the
argument list of rte_service_set_stats_enable(int enable)
+/export/dpdk.org/lib/librte_eal/common/include/rte_service.h:346:
warning: The following parameters of rte_service_set_stats_enable(int
enable) are not documented:
+  parameter 'enable'                                                           
+/export/dpdk.org/lib/librte_eal/common/include/rte_service.h:349:
warning: argument 'The' of command @param is not found in the argument
list of rte_service_lcore_list(uint32_t array[], uint32_t n)
+/export/dpdk.org/lib/librte_eal/common/include/rte_service.h:367:
warning: The following parameters of rte_service_lcore_list(uint32_t
array[], uint32_t n) are not documented:
+  parameter 'n'      

command to reproduce:
./devtools/test-build.sh -j8 x86_64-native-linuxapp-gcc+shared x86_64-native-linuxapp-gcc+debug



>  } DPDK_17.08;
> diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
> index f2fe052..942f03c 100644
> --- a/lib/librte_eal/common/Makefile
> +++ b/lib/librte_eal/common/Makefile
> @@ -41,6 +41,7 @@ INC += rte_eal_memconfig.h rte_malloc_heap.h
>  INC += rte_hexdump.h rte_devargs.h rte_bus.h rte_dev.h rte_vdev.h
>  INC += rte_pci_dev_feature_defs.h rte_pci_dev_features.h
>  INC += rte_malloc.h rte_keepalive.h rte_time.h
> +INC += rte_service.h rte_service_private.h

IMO, We don't need to expose rte_service_private.h to application. If
you agree, add the following or similar change

diff --git a/drivers/event/sw/Makefile b/drivers/event/sw/Makefile
index 857a87cc5..442652e93 100644
--- a/drivers/event/sw/Makefile
+++ b/drivers/event/sw/Makefile
@@ -36,6 +36,8 @@ LIB = librte_pmd_sw_event.a
 # build flags
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common/include/
+
 # for older GCC versions, allow us to initialize an event using
 # designated initializers.
 ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y)
diff --git a/lib/librte_eal/common/Makefile
b/lib/librte_eal/common/Makefile
index 942f03cdd..ea6c1f8f1 100644
--- a/lib/librte_eal/common/Makefile
+++ b/lib/librte_eal/common/Makefile
@@ -41,7 +41,7 @@ INC += rte_eal_memconfig.h rte_malloc_heap.h
 INC += rte_hexdump.h rte_devargs.h rte_bus.h rte_dev.h rte_vdev.h
 INC += rte_pci_dev_feature_defs.h rte_pci_dev_features.h
 INC += rte_malloc.h rte_keepalive.h rte_time.h
-INC += rte_service.h rte_service_private.h
+INC += rte_service.h

diff --git a/test/test/Makefile b/test/test/Makefile
index 42d9a49e2..b603b6563 100644
--- a/test/test/Makefile
+++ b/test/test/Makefile
@@ -152,6 +152,7 @@ SRCS-y += test_version.c
 SRCS-y += test_func_reentrancy.c
 
 SRCS-y += test_service_cores.c
+CFLAGS_test_service_cores.o += -I$(RTE_SDK)/lib/librte_eal/common/include/
 
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline.c
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_num.



>  
>  GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h
>  GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h
> diff --git a/lib/librte_eal/common/eal_common_lcore.c b/lib/librte_eal/common/eal_common_lcore.c
> index 84fa0cb..0db1555 100644
> --- a/lib/librte_eal/common/eal_common_lcore.c
> +++ b/lib/librte_eal/common/eal_common_lcore.c
> @@ -81,6 +81,7 @@ rte_eal_cpu_init(void)
>  
>  		/* By default, each detected core is enabled */
>  		config->lcore_role[lcore_id] = ROLE_RTE;
> +		lcore_config[lcore_id].core_role = ROLE_RTE;
>  		lcore_config[lcore_id].core_id = eal_cpu_core_id(lcore_id);
>  		lcore_config[lcore_id].socket_id = eal_cpu_socket_id(lcore_id);
>  		if (lcore_config[lcore_id].socket_id >= RTE_MAX_NUMA_NODES) {
> diff --git a/lib/librte_eal/common/include/rte_eal.h b/lib/librte_eal/common/include/rte_eal.h
> index abf020b..4dd0518 100644
> --- a/lib/librte_eal/common/include/rte_eal.h
> +++ b/lib/librte_eal/common/include/rte_eal.h
> @@ -61,6 +61,7 @@ extern "C" {
>  enum rte_lcore_role_t {
>  	ROLE_RTE,
>  	ROLE_OFF,
> +	ROLE_SERVICE,
>  };
>  
> +
> +#define SERVICE_F_REGISTERED 0
> +
> +/* runstates for services and lcores, denoting if they are active or not */
> +#define RUNSTATE_STOPPED 0
> +#define RUNSTATE_RUNNING 1
> +
> +/* internal representation of a service */
> +struct rte_service_spec_impl {
> +	/* public part of the struct */
> +	struct rte_service_spec spec;
> +
> +	/* atomic lock that when set indicates a service core is currently
> +	 * running this service callback. When not set, a core may take the
> +	 * lock and then run the service callback.
> +	 */
> +	rte_atomic32_t execute_lock;
> +
> +	/* API set/get-able variables */
> +	int32_t runstate;
> +	uint8_t internal_flags;
> +
> +	/* per service statistics */
> +	uint32_t num_mapped_cores;
> +	uint64_t calls;
> +	uint64_t cycles_spent;
> +} __rte_cache_aligned;
> +
> +/* the internal values of a service core */
> +struct core_state {

Change to lcore_state.

> +	/* map of services IDs are run on this core */
> +	uint64_t service_mask;
> +	uint8_t runstate; /* running or stopped */
> +	uint8_t is_service_core; /* set if core is currently a service core */
> +	uint8_t collect_statistics; /* if set, measure cycle counts */
> +
> +	/* extreme statistics */
> +	uint64_t calls_per_service[RTE_SERVICE_NUM_MAX];
> +} __rte_cache_aligned;
> +
> +static uint32_t rte_service_count;
> +static struct rte_service_spec_impl *rte_services;
> +static struct core_state *cores_state;
> +static uint32_t rte_service_library_initialized;
> +
> +int32_t rte_service_init(void)
> +{
> +	if (rte_service_library_initialized) {
> +		printf("service library init() called, init flag %d\n",
> +			rte_service_library_initialized);
> +		return -EALREADY;
> +	}
> +
> +	rte_services = rte_calloc("rte_services", RTE_SERVICE_NUM_MAX,
> +			sizeof(struct rte_service_spec_impl),
> +			RTE_CACHE_LINE_SIZE);
> +	if (!rte_services) {
> +		printf("error allocating rte services array\n");
> +		return -ENOMEM;
> +	}
> +
> +	cores_state = rte_calloc("rte_service_core_states", RTE_MAX_LCORE,
> +			sizeof(struct core_state), RTE_CACHE_LINE_SIZE);
> +	if (!cores_state) {
> +		printf("error allocating core states array\n");
> +		return -ENOMEM;
> +	}
> +
> +	int i;
> +	int count = 0;
> +	struct rte_config *cfg = rte_eal_get_configuration();
> +	for (i = 0; i < RTE_MAX_LCORE; i++) {
> +		if (lcore_config[i].core_role == ROLE_SERVICE) {
> +			if ((unsigned int)i == cfg->master_lcore)
> +				continue;
> +			rte_service_lcore_add(i);
> +			count++;
> +		}
> +	}
> +
> +	rte_service_library_initialized = 1;
> +	return 0;
> +}
> +
> +void rte_service_set_stats_enable(int enabled)

IMO, It should be per service  i.e
rte_service_set_stats_enable(const struct rte_service_spec *spec, int enable)

> +{
> +	uint32_t i;
> +	for (i = 0; i < RTE_MAX_LCORE; i++)
> +		cores_state[i].collect_statistics = enabled;
> +}
> +
> +/* returns 1 if service is registered and has not been unregistered
> + * Returns 0 if service never registered, or has been unregistered
> + */
> +static inline int
> +service_valid(uint32_t id) {
> +	return !!(rte_services[id].internal_flags &
> +		 (1 << SERVICE_F_REGISTERED));
> +}
> +
> +uint32_t
> +rte_service_get_count(void)
> +{
> +	return rte_service_count;
> +}
> +
> +struct rte_service_spec *
> +rte_service_get_by_id(uint32_t id)
> +{
> +	struct rte_service_spec *service = NULL;
> +	if (id < rte_service_count)
> +		service = (struct rte_service_spec *)&rte_services[id];
> +
> +	return service;
> +}
> +
> +struct rte_service_spec *rte_service_get_by_name(const char *name)
> +{
> +	struct rte_service_spec *service = NULL;
> +	int i;
> +	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
> +		if (service_valid(i) &&
> +				strcmp(name, rte_services[i].spec.name) == 0) {
> +			service = (struct rte_service_spec *)&rte_services[i];
> +			break;
> +		}
> +	}
> +
> +	return service;
> +}
> +
> +const char *
> +rte_service_get_name(const struct rte_service_spec *service)
> +{
> +	return service->name;
> +}
> +
> +int32_t
> +rte_service_probe_capability(const struct rte_service_spec *service,
> +			     uint32_t capability)
> +{
> +	return service->capabilities & capability;
> +}
> +
> +int32_t
> +rte_service_is_running(const struct rte_service_spec *spec)
> +{
> +	const struct rte_service_spec_impl *impl =
> +		(const struct rte_service_spec_impl *)spec;
> +	if (!impl)
> +		return -EINVAL;
> +
> +	return (impl->runstate == RUNSTATE_RUNNING) &&
> +		(impl->num_mapped_cores > 0);
> +}
> +
> +int32_t
> +rte_service_register(const struct rte_service_spec *spec)
> +{
> +	uint32_t i;
> +	int32_t free_slot = -1;
> +
> +	if (spec->callback == NULL || strlen(spec->name) == 0)
> +		return -EINVAL;
> +
> +	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
> +		if (!service_valid(i)) {
> +			free_slot = i;
> +			break;
> +		}
> +	}
> +
> +	if ((free_slot < 0) || (i == RTE_SERVICE_NUM_MAX))
> +		return -ENOSPC;
> +
> +	struct rte_service_spec_impl *s = &rte_services[free_slot];
> +	s->spec = *spec;
> +	s->internal_flags |= (1 << SERVICE_F_REGISTERED);
> +
> +	rte_smp_wmb();
> +	rte_service_count++;
> +
> +	return 0;
> +}
> +
> +int32_t
> +rte_service_unregister(struct rte_service_spec *spec)
> +{
> +	struct rte_service_spec_impl *s = NULL;
> +	struct rte_service_spec_impl *spec_impl =
> +		(struct rte_service_spec_impl *)spec;
> +
> +	uint32_t i;
> +	uint32_t service_id;
> +	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
> +		if (&rte_services[i] == spec_impl) {
> +			s = spec_impl;
> +			service_id = i;
> +			break;
> +		}
> +	}
> +
> +	if (!s)
> +		return -EINVAL;
> +
> +	rte_service_count--;
> +	rte_smp_wmb();
> +
> +	s->internal_flags &= ~(1 << SERVICE_F_REGISTERED);
> +
> +	for (i = 0; i < RTE_MAX_LCORE; i++)
> +		cores_state[i].service_mask &= ~(1 << service_id);
> +
> +	memset(&rte_services[service_id], 0,
> +			sizeof(struct rte_service_spec_impl));
> +
> +	return 0;
> +}
> +
> +int32_t
> +rte_service_start(struct rte_service_spec *service)
> +{
> +	struct rte_service_spec_impl *s =
> +		(struct rte_service_spec_impl *)service;
> +	s->runstate = RUNSTATE_RUNNING;
> +	rte_smp_wmb();
> +	return 0;
> +}
> +
> +int32_t
> +rte_service_stop(struct rte_service_spec *service)
> +{
> +	struct rte_service_spec_impl *s =
> +		(struct rte_service_spec_impl *)service;
> +	s->runstate = RUNSTATE_STOPPED;
> +	rte_smp_wmb();
> +	return 0;
> +}
> +
> +static int32_t
> +rte_service_runner_func(void *arg)
> +{
> +	RTE_SET_USED(arg);
> +	uint32_t i;
> +	const int lcore = rte_lcore_id();
> +	struct core_state *cs = &cores_state[lcore];
> +
> +	while (cores_state[lcore].runstate == RUNSTATE_RUNNING) {
> +		const uint64_t service_mask = cs->service_mask;
> +		for (i = 0; i < rte_service_count; i++) {
> +			struct rte_service_spec_impl *s = &rte_services[i];
> +			if (s->runstate != RUNSTATE_RUNNING ||
> +					!(service_mask & (1 << i)))
> +				continue;
> +
> +			/* check if this is the only core mapped, else use
> +			 * atomic to serialize cores mapped to this service
> +			 */
> +			uint32_t *lock = (uint32_t *)&s->execute_lock;
> +			if ((s->spec.capabilities & RTE_SERVICE_CAP_MT_SAFE) ||
> +					(s->num_mapped_cores == 1 ||
> +					rte_atomic32_cmpset(lock, 0, 1))) {
> +				void *userdata = s->spec.callback_userdata;
> +
> +				if (cs->collect_statistics) {
> +					uint64_t start = rte_rdtsc();
> +					s->spec.callback(userdata);
> +					uint64_t end = rte_rdtsc();
> +					s->cycles_spent += end - start;
> +					cs->calls_per_service[i]++;
> +					s->calls++;
> +				} else
> +					s->spec.callback(userdata);
> +
> +				if ((s->spec.capabilities &
> +						RTE_SERVICE_CAP_MT_SAFE) == 0 &&
> +						s->num_mapped_cores > 1)

How about computing the non rte_atomic32_cmpset() mode value first and
using in both place i.e here and in the top "if" loop

	const int need_cmpset = (s->spec.capabilities & RTE_SERVICE_CAP_MT_SAFE)...
	if (need_cmpset || rte_atomic32_cmpset(lock, 0, 1))
	..
	if (need_cmpset)
		rte_atomic32_clear()..


> +					rte_atomic32_clear(&s->execute_lock);
> +			}
> +		}
> +
> +		rte_smp_rmb();
> +	}
> +
> +	lcore_config[lcore].state = WAIT;
> +
> +	return 0;
> +}
> +
> +int32_t
> +rte_service_lcore_count(void)
> +{
> +	int32_t count = 0;
> +	uint32_t i;
> +	for (i = 0; i < RTE_MAX_LCORE; i++)
> +		count += cores_state[i].is_service_core;
> +	return count;
> +}
> +
> +int32_t
> +rte_service_lcore_list(uint32_t array[], uint32_t n)
> +{
> +	uint32_t count = rte_service_lcore_count();
> +	if (count > n)
> +		return -ENOMEM;
> +
> +	if (!array)
> +		return -EINVAL;
> +
> +	uint32_t i;
> +	uint32_t idx = 0;
> +	for (i = 0; i < RTE_MAX_LCORE; i++) {
> +		struct core_state *cs = &cores_state[i];
> +		if (cs->is_service_core) {
> +			array[idx] = i;
> +			idx++;
> +		}
> +	}
> +
> +	return count;
> +}
> +
> +int32_t
> +rte_service_set_default_mapping(void)
> +{
> +	/* create a default mapping from cores to services, then start the
> +	 * services to make them transparent to unaware applications.
> +	 */
> +	uint32_t i;
> +	int ret;
> +	uint32_t count = rte_service_get_count();
> +
> +	int32_t lcore_iter = 0;
> +	uint32_t ids[RTE_MAX_LCORE];
> +	int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE);
> +
> +	for (i = 0; i < count; i++) {
> +		struct rte_service_spec *s = rte_service_get_by_id(i);
> +		if (!s)
> +			return -EINVAL;
> +
> +		/* if no lcores available as services cores, don't setup map.
> +		 * This means app logic must add cores, and setup mappings
> +		 */
> +		if (lcore_count > 0) {


> +			/* do 1:1 core mapping here, with each service getting
> +			 * assigned a single core by default. Adding multiple
> +			 * services should multiplex to a single core, or 1:1
> +			 * if services == cores
> +			 */
> +			ret = rte_service_enable_on_lcore(s, ids[lcore_iter]);
> +			if (ret)
> +				return -ENODEV;
> +		}
> +
> +		lcore_iter++;
> +		if (lcore_iter >= lcore_count)
> +			lcore_iter = 0;
> +
> +		ret = rte_service_start(s);

IMO, we don't need to start the service if lcore_count == 0. How about
moving the "if (lcore_count > 0)" check on top of for the loop and exist
from the function if lcore_count == 0.


> +		if (ret)
> +			return -ENOEXEC;
> +	}
> +
> +	return 0;
> +}
> +
> +static int32_t
> +service_update(struct rte_service_spec *service, uint32_t lcore,
> +		uint32_t *set, uint32_t *enabled)
> +{
> +	uint32_t i;
> +	int32_t sid = -1;
> +
> +	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
> +		if ((struct rte_service_spec *)&rte_services[i] == service &&
> +				service_valid(i)) {
> +			sid = i;
> +			break;
> +		}
> +	}
> +
> +	if (sid == -1 || lcore >= RTE_MAX_LCORE)
> +		return -EINVAL;
> +
> +	if (!cores_state[lcore].is_service_core)
> +		return -EINVAL;
> +
> +	if (set) {
> +		if (*set) {
> +			cores_state[lcore].service_mask |=  (1 << sid);
> +			rte_services[sid].num_mapped_cores++;
> +		} else {
> +			cores_state[lcore].service_mask &= ~(1 << sid);
> +			rte_services[sid].num_mapped_cores--;
> +		}
> +	}
> +
> +	if (enabled)
> +		*enabled = (cores_state[lcore].service_mask & (1 << sid));
> +
> +	rte_smp_wmb();
> +
> +	return 0;
> +}
> +
> +int32_t rte_service_get_enabled_on_lcore(struct rte_service_spec *service,
> +					uint32_t lcore)
> +{
> +	uint32_t enabled;
> +	int ret = service_update(service, lcore, 0, &enabled);
> +	if (ret == 0)
> +		return enabled;
> +	return -EINVAL;
> +}
> +
> +int32_t
> +rte_service_enable_on_lcore(struct rte_service_spec *service, uint32_t lcore)
> +{
> +	uint32_t on = 1;
> +	return service_update(service, lcore, &on, 0);
> +}
> +
> +int32_t
> +rte_service_disable_on_lcore(struct rte_service_spec *service, uint32_t lcore)
> +{
> +	uint32_t off = 0;
> +	return service_update(service, lcore, &off, 0);
> +}
> +
> +int32_t rte_service_lcore_reset_all(void)
> +{
> +	/* loop over cores, reset all to mask 0 */
> +	uint32_t i;
> +	for (i = 0; i < RTE_MAX_LCORE; i++) {
> +		cores_state[i].service_mask = 0;
> +		cores_state[i].is_service_core = 0;
> +		cores_state[i].runstate = RUNSTATE_STOPPED;
> +	}
> +	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++)
> +		rte_services[i].num_mapped_cores = 0;
> +
> +	rte_smp_wmb();
> +
> +	return 0;
> +}
> +
> +static void
> +set_lcore_state(uint32_t lcore, int32_t state)
> +{
> +	/* mark core state in hugepage backed config */
> +	struct rte_config *cfg = rte_eal_get_configuration();
> +	cfg->lcore_role[lcore] = state;
> +
> +	/* mark state in process local lcore_config */
> +	lcore_config[lcore].core_role = state;
> +
> +	/* update per-lcore optimized state tracking */
> +	cores_state[lcore].is_service_core = (state == ROLE_SERVICE);
> +}
> +
> +int32_t
> +rte_service_lcore_add(uint32_t lcore)
> +{
> +	if (lcore >= RTE_MAX_LCORE)
> +		return -EINVAL;
> +	if (cores_state[lcore].is_service_core)
> +		return -EALREADY;
> +
> +	set_lcore_state(lcore, ROLE_SERVICE);
> +
> +	/* ensure that after adding a core the mask and state are defaults */
> +	cores_state[lcore].service_mask = 0;
> +	cores_state[lcore].runstate = RUNSTATE_STOPPED;

If worker core can call rte_service_lcore_add() then add rte_smp_wmb()
here. Applies to rte_service_lcore_del() as well.

Re: [PATCH v4 1/7] service cores: header and implementation

[Thomas Monjalon]

11/07/2017 10:29, Jerin Jacob:
> IMO, We don't need to expose rte_service_private.h to application. If
> you agree, add the following or similar change

If it must not be exposed, the file should not have the prefix rte_
In doc/api/doxy-api.conf, every files with rte_ prefix will be processed
for doxygen documentation:
	FILE_PATTERNS = rte_*.h

Re: [PATCH v4 1/7] service cores: header and implementation

[Van Haaren, Harry]

> From: Thomas Monjalon [mailto:thomas@monjalon.net]
> Sent: Tuesday, July 11, 2017 10:55 AM
> To: Jerin Jacob <jerin.jacob@caviumnetworks.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>
> Cc: dev@dpdk.org; Wiles, Keith <keith.wiles@intel.com>; Richardson, Bruce
> <bruce.richardson@intel.com>
> Subject: Re: [PATCH v4 1/7] service cores: header and implementation
> 
> 11/07/2017 10:29, Jerin Jacob:
> > IMO, We don't need to expose rte_service_private.h to application. If
> > you agree, add the following or similar change
> 
> If it must not be exposed, the file should not have the prefix rte_
> In doc/api/doxy-api.conf, every files with rte_ prefix will be processed
> for doxygen documentation:
> 	FILE_PATTERNS = rte_*.h


The service registration API should be exposed to the application.

Imagine a use case where the application wants to run services *and* an application specific function on the same core.  In the current implementation this is possible, as the application can register a service. The app then configures all services (including its own "app-service") to run on a service lcore.

If we hide the service registration from the application, we make it impossible for the application to multiplex services and application specific workloads on a single core.


I strongly prefer of leaving the header as is. Given we have EXPERIMENTAL tag, ABI/API are not a concern until later - we have time to figure out if the service-registration API is good enough in current form, before we commit to it.

I'll send v5 asap with headers as is.

Re: [PATCH v4 1/7] service cores: header and implementation

[Jerin Jacob]

-----Original Message-----
> Date: Tue, 11 Jul 2017 12:32:58 +0000
> From: "Van Haaren, Harry" <harry.van.haaren@intel.com>
> To: Thomas Monjalon <thomas@monjalon.net>, Jerin Jacob
>  <jerin.jacob@caviumnetworks.com>
> CC: "dev@dpdk.org" <dev@dpdk.org>, "Wiles, Keith" <keith.wiles@intel.com>,
>  "Richardson, Bruce" <bruce.richardson@intel.com>
> Subject: RE: [PATCH v4 1/7] service cores: header and implementation
> 
> > From: Thomas Monjalon [mailto:thomas@monjalon.net]
> > Sent: Tuesday, July 11, 2017 10:55 AM
> > To: Jerin Jacob <jerin.jacob@caviumnetworks.com>; Van Haaren, Harry
> > <harry.van.haaren@intel.com>
> > Cc: dev@dpdk.org; Wiles, Keith <keith.wiles@intel.com>; Richardson, Bruce
> > <bruce.richardson@intel.com>
> > Subject: Re: [PATCH v4 1/7] service cores: header and implementation
> > 
> > 11/07/2017 10:29, Jerin Jacob:
> > > IMO, We don't need to expose rte_service_private.h to application. If
> > > you agree, add the following or similar change
> > 
> > If it must not be exposed, the file should not have the prefix rte_
> > In doc/api/doxy-api.conf, every files with rte_ prefix will be processed
> > for doxygen documentation:
> > 	FILE_PATTERNS = rte_*.h
> 
> 
> The service registration API should be exposed to the application.
> 
> Imagine a use case where the application wants to run services *and* an application specific function on the same core.  In the current implementation this is possible, as the application can register a service. The app then configures all services (including its own "app-service") to run on a service lcore.
> 
> If we hide the service registration from the application, we make it impossible for the application to multiplex services and application specific workloads on a single core.

Then we could move the registration functions to service.h.
IMO, It does not look correct if we expose _prviate.h to application or
we could rename to service_component.h or something like that.

> 
> 
> I strongly prefer of leaving the header as is. Given we have EXPERIMENTAL tag, ABI/API are not a concern until later - we have time to figure out if the service-registration API is good enough in current form, before we commit to it.
> 
> I'll send v5 asap with headers as is.

Re: [PATCH v4 1/7] service cores: header and implementation

[Van Haaren, Harry]

> From: Jerin Jacob [mailto:jerin.jacob@caviumnetworks.com]
> Sent: Tuesday, July 11, 2017 1:45 PM
> To: Van Haaren, Harry <harry.van.haaren@intel.com>
> Cc: Thomas Monjalon <thomas@monjalon.net>; dev@dpdk.org; Wiles, Keith
> <keith.wiles@intel.com>; Richardson, Bruce <bruce.richardson@intel.com>
> Subject: Re: [PATCH v4 1/7] service cores: header and implementation

<snip>

> > The service registration API should be exposed to the application.
> >
> > Imagine a use case where the application wants to run services *and* an application
> specific function on the same core.  In the current implementation this is possible, as
> the application can register a service. The app then configures all services (including
> its own "app-service") to run on a service lcore.
> >
> > If we hide the service registration from the application, we make it impossible for the
> application to multiplex services and application specific workloads on a single core.
> 
> Then we could move the registration functions to service.h.
> IMO, It does not look correct if we expose _prviate.h to application or
> we could rename to service_component.h or something like that.


Fair enough - I will rename to rte_service_component.h for v5. Thanks for input!

Re: [PATCH v4 1/7] service cores: header and implementation

[Van Haaren, Harry]

<lots of snips to make responses consumable!>

> From: Jerin Jacob [mailto:jerin.jacob@caviumnetworks.com]
> To: Van Haaren, Harry <harry.van.haaren@intel.com>
> Cc: dev@dpdk.org; thomas@monjalon.net; Wiles, Keith <keith.wiles@intel.com>; Richardson,
> Bruce <bruce.richardson@intel.com>
> Subject: Re: [PATCH v4 1/7] service cores: header and implementation
> 
<snip>
> 
> Remove above info from the git commit.

Done


> Fix the below mentioned documentation warning.
> 
> +/export/dpdk.org/lib/librte_eal/common/include/rte_service.h:338:
> warning: argument 'enabled' of command @param is not found in the
> argument list of rte_service_set_stats_enable(int enable)
> +/export/dpdk.org/lib/librte_eal/common/include/rte_service.h:346:
> warning: The following parameters of rte_service_set_stats_enable(int
> enable) are not documented:
> +  parameter 'enable'
> +/export/dpdk.org/lib/librte_eal/common/include/rte_service.h:349:
> warning: argument 'The' of command @param is not found in the argument
> list of rte_service_lcore_list(uint32_t array[], uint32_t n)
> +/export/dpdk.org/lib/librte_eal/common/include/rte_service.h:367:
> warning: The following parameters of rte_service_lcore_list(uint32_t
> array[], uint32_t n) are not documented:
> +  parameter 'n'

Done


> command to reproduce:
> ./devtools/test-build.sh -j8 x86_64-native-linuxapp-gcc+shared x86_64-native-linuxapp-
> gcc+debug

Thanks - noted.


> > +
> > +/* the internal values of a service core */
> > +struct core_state {
> 
> Change to lcore_state.

Done.


> > +
> > +void rte_service_set_stats_enable(int enabled)
> 
> IMO, It should be per service  i.e
> rte_service_set_stats_enable(const struct rte_service_spec *spec, int enable)

Improved service library to handle statistics collection on a per service basis.


> > +			/* check if this is the only core mapped, else use
> > +			 * atomic to serialize cores mapped to this service
> > +			 */
> > +			uint32_t *lock = (uint32_t *)&s->execute_lock;
> > +			if ((s->spec.capabilities & RTE_SERVICE_CAP_MT_SAFE) ||
> > +					(s->num_mapped_cores == 1 ||
> > +					rte_atomic32_cmpset(lock, 0, 1))) {
> > +				void *userdata = s->spec.callback_userdata;
> > +
> > +				if (cs->collect_statistics) {
> > +					uint64_t start = rte_rdtsc();
> > +					s->spec.callback(userdata);
> > +					uint64_t end = rte_rdtsc();
> > +					s->cycles_spent += end - start;
> > +					cs->calls_per_service[i]++;
> > +					s->calls++;
> > +				} else
> > +					s->spec.callback(userdata);
> > +
> > +				if ((s->spec.capabilities &
> > +						RTE_SERVICE_CAP_MT_SAFE) == 0 &&
> > +						s->num_mapped_cores > 1)
> 
> How about computing the non rte_atomic32_cmpset() mode value first and
> using in both place i.e here and in the top "if" loop
> 
> 	const int need_cmpset = (s->spec.capabilities & RTE_SERVICE_CAP_MT_SAFE)...
> 	if (need_cmpset || rte_atomic32_cmpset(lock, 0, 1))
> 	..
> 	if (need_cmpset)
> 		rte_atomic32_clear()..


Yes good idea. Indeed I wasn't happy with that, this is a good fix.
The checks to detect if we need_cmpset are a little complex, but it's the only solution I see.

I've added another unit test to verify both MT safe and MT unsafe callback operations.


> > +int32_t
> > +rte_service_set_default_mapping(void)
> > +{
> > +	/* create a default mapping from cores to services, then start the
> > +	 * services to make them transparent to unaware applications.
> > +	 */
> > +	uint32_t i;
> > +	int ret;
> > +	uint32_t count = rte_service_get_count();
> > +
> > +	int32_t lcore_iter = 0;
> > +	uint32_t ids[RTE_MAX_LCORE];
> > +	int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE);
> > +
> > +	for (i = 0; i < count; i++) {
> > +		struct rte_service_spec *s = rte_service_get_by_id(i);
> > +		if (!s)
> > +			return -EINVAL;
> > +
> > +		/* if no lcores available as services cores, don't setup map.
> > +		 * This means app logic must add cores, and setup mappings
> > +		 */
> > +		if (lcore_count > 0) {
> 
> 
> > +			/* do 1:1 core mapping here, with each service getting
> > +			 * assigned a single core by default. Adding multiple
> > +			 * services should multiplex to a single core, or 1:1
> > +			 * if services == cores
> > +			 */
> > +			ret = rte_service_enable_on_lcore(s, ids[lcore_iter]);
> > +			if (ret)
> > +				return -ENODEV;
> > +		}
> > +
> > +		lcore_iter++;
> > +		if (lcore_iter >= lcore_count)
> > +			lcore_iter = 0;
> > +
> > +		ret = rte_service_start(s);
> 
> IMO, we don't need to start the service if lcore_count == 0. How about
> moving the "if (lcore_count > 0)" check on top of for the loop and exist
> from the function if lcore_count == 0.

Good point, added   if() check for lcore_count at start of function, and return if no service cores are available.


> > +
> > +int32_t
> > +rte_service_lcore_add(uint32_t lcore)
> > +{
> > +	if (lcore >= RTE_MAX_LCORE)
> > +		return -EINVAL;
> > +	if (cores_state[lcore].is_service_core)
> > +		return -EALREADY;
> > +
> > +	set_lcore_state(lcore, ROLE_SERVICE);
> > +
> > +	/* ensure that after adding a core the mask and state are defaults */
> > +	cores_state[lcore].service_mask = 0;
> > +	cores_state[lcore].runstate = RUNSTATE_STOPPED;
> 
> If worker core can call rte_service_lcore_add() then add rte_smp_wmb()
> here. Applies to rte_service_lcore_del() as well.

Added barriers to both add() and del().

Re: [PATCH v4 2/7] service cores: EAL init changes

[Van Haaren, Harry]

> From: Jerin Jacob [mailto:jerin.jacob@caviumnetworks.com]
> Sent: Tuesday, July 11, 2017 8:42 AM
> To: Van Haaren, Harry <harry.van.haaren@intel.com>
> Cc: dev@dpdk.org; thomas@monjalon.net; Wiles, Keith <keith.wiles@intel.com>; Richardson,
> Bruce <bruce.richardson@intel.com>
> Subject: Re: [PATCH v4 2/7] service cores: EAL init changes

<snip>

> > +	/* initialize default services configuration */
> > +	uint32_t service_cores[RTE_MAX_LCORE];
> > +	int count = rte_service_lcore_list(service_cores, RTE_MAX_LCORE);
> > +	for (i = 0; i < count; i++)
> > +		rte_service_lcore_start(service_cores[i]);
> > +	ret = rte_service_set_default_mapping();
> > +	if (ret) {
> > +		rte_errno = ENOEXEC;
> > +		return -1;
> > +	}
> 
> How about moving, rte_service_lcore_start() inside
> rte_service_set_default_mapping() so that rte_eal_init() level change will be
> less in linuxapp and bsdapp?(and both changes are tightly coupled too).
> 
> You could change the function name to rte_service_enable_default_mapping()
> or something like that to include rte_service_lcore_start() start change.

Good idea - done. Does indeed make things cleaner - thanks!

> With that change:
> Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

Added to patch! Cheers, -Harry

[PATCH v5 0/7] service cores: cover letter

[Harry van Haaren]

This patchset introduces service cores to DPDK. A service core
is an lcore that performs functions to abstract away details of
differences in environment of the application.

An example is using the eventdev API, where either a software or hardware
PMD performs scheduling. In the case of the software PMD an lcore is
required to perform scheduling, which means application logic would have
to be aware of the PMD running under the API. To abstract away the
differences in HW / SW PMDs, service cores can run the SW PMD service
without application logic specifying the exact cores to use. Note that
eventdev is only one API that benefits; timers, interrupts handling,
statistics and monitoring, and a range of other infrastructure that
requires a slice of CPU time may all benefit from service cores.

The application is not obliged to manually use the service cores API,
however if an application wishes to use the service cores API for fine
grained control over how the services are run, this is possible. Deciding
between a performance threading-profile and scaled-down profile can be
achieved by advanced usage of service cores and setting the lcore mappings.

Patch 5/7 shows how a PMD can register a service to run a function. This
is then available (along with any other registered services) to be run by
the service cores. Patches 6/7 and 7/7 add documentation, and claim
maintainership.

Regards, -Harry

v5:
Jerin:
- Fix documentation warnings
- Rename variables to better names
- Enable statistics per-service
- Improve atomic operation flag checks
- Reworked function to rte_service_start_with_defaults()
- Added memory barriers to lcore_add() and lcore_del()
- Simplified EAL code, reduced duplication and makes it more maintainable

Jerin/Thomas:
- Rename component header to rte_service_component.h

v4:
- Range of fixes as suggested by Jerin
- Improved unit tests, ensuring ex-service cores become available to app
- Added functions to EXPERIMENTAL tag in .map files (Thomas)
- Added @warning experimental notes to Doxygen API documentation (Thomas)
- Various smaller fixes / cleanups
- See commit notes for details

v3:
- Added docs
- Added release notes
- Updated maintainers file
- Compile checks with devtools/test-build.sh
- Validated patches apply to latest dpdk/master
- Based on discussion, rte_service_iterate() is *not* included,
  but could be adding at a later date if use-cases require it.
- Future work includes enabling the eventdev_pipeline sample app, but there
  is still some churn there to enable both HW/SW PMDs seamlessly. Once sample
  app is enabled a service core walk-through with that sample app can be added
  to the docs, to provide a tutorial on service-core usage.


Harry van Haaren (7):
  service cores: header and implementation
  service cores: EAL init changes
  service cores: coremask parsing
  service cores: add unit tests
  event/sw: enable SW PMD with service capability
  doc: add service cores to doc and release notes
  maintainers: claim service cores

 MAINTAINERS                                        |   7 +
 doc/api/doxy-api-index.md                          |   1 +
 doc/guides/eventdevs/sw.rst                        |   4 +-
 doc/guides/prog_guide/index.rst                    |   1 +
 doc/guides/prog_guide/service_cores.rst            |  81 +++
 doc/guides/rel_notes/release_17_08.rst             |   8 +
 drivers/event/sw/sw_evdev.c                        |  32 +
 drivers/event/sw/sw_evdev.h                        |   3 +
 lib/librte_eal/bsdapp/eal/Makefile                 |   1 +
 lib/librte_eal/bsdapp/eal/eal.c                    |  18 +
 lib/librte_eal/bsdapp/eal/rte_eal_version.map      |  23 +
 lib/librte_eal/common/Makefile                     |   1 +
 lib/librte_eal/common/eal_common_lcore.c           |   1 +
 lib/librte_eal/common/eal_common_options.c         |  91 ++-
 lib/librte_eal/common/include/rte_eal.h            |   4 +
 lib/librte_eal/common/include/rte_lcore.h          |   3 +-
 lib/librte_eal/common/include/rte_service.h        | 387 +++++++++++
 .../common/include/rte_service_component.h         | 144 +++++
 lib/librte_eal/common/rte_service.c                | 704 +++++++++++++++++++++
 lib/librte_eal/linuxapp/eal/Makefile               |   1 +
 lib/librte_eal/linuxapp/eal/eal.c                  |  18 +
 lib/librte_eal/linuxapp/eal/eal_thread.c           |   9 +-
 lib/librte_eal/linuxapp/eal/rte_eal_version.map    |  23 +
 test/test/Makefile                                 |   2 +
 test/test/test_service_cores.c                     | 599 ++++++++++++++++++
 25 files changed, 2162 insertions(+), 4 deletions(-)
 create mode 100644 doc/guides/prog_guide/service_cores.rst
 create mode 100644 lib/librte_eal/common/include/rte_service.h
 create mode 100644 lib/librte_eal/common/include/rte_service_component.h
 create mode 100644 lib/librte_eal/common/rte_service.c
 create mode 100644 test/test/test_service_cores.c

-- 
2.7.4

[PATCH v5 1/7] service cores: header and implementation

[Harry van Haaren]

Add header files, update .map files with new service
functions, and add the service header to the doxygen
for building.

This service header API allows DPDK to use services as
a concept of something that requires CPU cycles. An example
is a PMD that runs in software to schedule events, where a
hardware version exists that does not require a CPU.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

---

v5:
- Improved service_set_stats_enable() to operate per service (Jerin)
- Fixed un-documented doxygen parameter (Jerin)
- Renamed cores_state to lcore_states (Jerin)
- Optimized atomic operations and flags (Jerin)
- Removed info about RFCs etc from commit message (Jerin)
- Add lcore_count check to default setup function and return early (Jerin)
- Add memory barriers to lcore_add() and lcore_del() (Jerin)
- Rename start function to rte_service_start_with_defaults() (Jerin)
- Rename header to rte_service_component.h (Jerin/Thomas)

v4:
- Fixed (unsigned) checkpatch error
- Fixed misleading-indentation/if { } brackets (checkpatch/Jerin)
- Fixed set function argument to be "enable" instead of "enabled" (Jerin)
- Improve doxygen comment for size of array in rte_service_core_list (Jerin)
- Fixed typos (Jerin)
- Optimized atomic clear after running service (Jerin)
- Added smp_rmb() at end of loop to re-load runstate / mapping (Jerin)
- Fix issue with lcore role not being adhered to (Jerin)
- Add experimental warnings for all service core functions (Thomas)
- Moved service core functions into EXPERIMENTAL section of .map (Thomas)
- Improve documentation of rte_service_lcore_reset_all() (Harry)

v3:
- None.

v2:
Thanks Jerin for review - below a list your suggested changes;
- Doxygen rename to "service cores" for consistency
- use lcore instead of core for function names
- Fix about 10 typos / seplling msitakse ;)
- Dix doxygen /** comments for functions
- Doxygen @param[out] improvements
- int8_t for socket_id to ordinary int
- Rename MACROS for readability
- Align structs to cache lines
- Allocate fastpath-used data from hugepages
- Added/fixed memory barriers for multi-core scheduling
- Add const to variables, and hoist above loop
- Optimize cmpset atomic if MT_SAFE or only one core mapped
- Statistics collection only when requested
- Add error check for array pointer
- Remove panic() calls from library
- Fix TODO notes from previous patchset

There are also some other changes;
- Checkpatch issues fixed
- .map file updates
- Add rte_service_get_by_name() function
---
 doc/api/doxy-api-index.md                          |   1 +
 lib/librte_eal/bsdapp/eal/Makefile                 |   1 +
 lib/librte_eal/bsdapp/eal/rte_eal_version.map      |  23 +
 lib/librte_eal/common/Makefile                     |   1 +
 lib/librte_eal/common/eal_common_lcore.c           |   1 +
 lib/librte_eal/common/include/rte_eal.h            |   4 +
 lib/librte_eal/common/include/rte_lcore.h          |   3 +-
 lib/librte_eal/common/include/rte_service.h        | 387 +++++++++++
 .../common/include/rte_service_component.h         | 144 +++++
 lib/librte_eal/common/rte_service.c                | 704 +++++++++++++++++++++
 lib/librte_eal/linuxapp/eal/Makefile               |   1 +
 lib/librte_eal/linuxapp/eal/eal_thread.c           |   9 +-
 lib/librte_eal/linuxapp/eal/rte_eal_version.map    |  23 +
 13 files changed, 1300 insertions(+), 2 deletions(-)
 create mode 100644 lib/librte_eal/common/include/rte_service.h
 create mode 100644 lib/librte_eal/common/include/rte_service_component.h
 create mode 100644 lib/librte_eal/common/rte_service.c

diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index 67594e1..e99e114 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -160,6 +160,7 @@ There are many libraries, so their headers may be grouped by topics:
   [common]             (@ref rte_common.h),
   [ABI compat]         (@ref rte_compat.h),
   [keepalive]          (@ref rte_keepalive.h),
+  [service cores]      (@ref rte_service.h),
   [device metrics]     (@ref rte_metrics.h),
   [bitrate statistics] (@ref rte_bitrate.h),
   [latency statistics] (@ref rte_latencystats.h),
diff --git a/lib/librte_eal/bsdapp/eal/Makefile b/lib/librte_eal/bsdapp/eal/Makefile
index a0f9950..05517a2 100644
--- a/lib/librte_eal/bsdapp/eal/Makefile
+++ b/lib/librte_eal/bsdapp/eal/Makefile
@@ -87,6 +87,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_malloc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_elem.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_heap.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_keepalive.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_service.c
 
 # from arch dir
 SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_cpuflags.c
diff --git a/lib/librte_eal/bsdapp/eal/rte_eal_version.map b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
index 381f895..480ad23 100644
--- a/lib/librte_eal/bsdapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/bsdapp/eal/rte_eal_version.map
@@ -209,5 +209,28 @@ EXPERIMENTAL {
 	rte_eal_devargs_parse;
 	rte_eal_hotplug_add;
 	rte_eal_hotplug_remove;
+	rte_service_disable_on_lcore;
+	rte_service_dump;
+	rte_service_enable_on_lcore;
+	rte_service_get_by_id;
+	rte_service_get_by_name;
+	rte_service_get_count;
+	rte_service_get_enabled_on_lcore;
+	rte_service_is_running;
+	rte_service_lcore_add;
+	rte_service_lcore_count;
+	rte_service_lcore_del;
+	rte_service_lcore_list;
+	rte_service_lcore_reset_all;
+	rte_service_lcore_start;
+	rte_service_lcore_stop;
+	rte_service_probe_capability;
+	rte_service_register;
+	rte_service_reset;
+	rte_service_set_stats_enable;
+	rte_service_start;
+	rte_service_start_with_defaults;
+	rte_service_stop;
+	rte_service_unregister;
 
 } DPDK_17.08;
diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile
index f2fe052..e8fd67a 100644
--- a/lib/librte_eal/common/Makefile
+++ b/lib/librte_eal/common/Makefile
@@ -41,6 +41,7 @@ INC += rte_eal_memconfig.h rte_malloc_heap.h
 INC += rte_hexdump.h rte_devargs.h rte_bus.h rte_dev.h rte_vdev.h
 INC += rte_pci_dev_feature_defs.h rte_pci_dev_features.h
 INC += rte_malloc.h rte_keepalive.h rte_time.h
+INC += rte_service.h rte_service_component.h
 
 GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h
 GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h
diff --git a/lib/librte_eal/common/eal_common_lcore.c b/lib/librte_eal/common/eal_common_lcore.c
index 84fa0cb..0db1555 100644
--- a/lib/librte_eal/common/eal_common_lcore.c
+++ b/lib/librte_eal/common/eal_common_lcore.c
@@ -81,6 +81,7 @@ rte_eal_cpu_init(void)
 
 		/* By default, each detected core is enabled */
 		config->lcore_role[lcore_id] = ROLE_RTE;
+		lcore_config[lcore_id].core_role = ROLE_RTE;
 		lcore_config[lcore_id].core_id = eal_cpu_core_id(lcore_id);
 		lcore_config[lcore_id].socket_id = eal_cpu_socket_id(lcore_id);
 		if (lcore_config[lcore_id].socket_id >= RTE_MAX_NUMA_NODES) {
diff --git a/lib/librte_eal/common/include/rte_eal.h b/lib/librte_eal/common/include/rte_eal.h
index 6b7c5ca..0e7363d 100644
--- a/lib/librte_eal/common/include/rte_eal.h
+++ b/lib/librte_eal/common/include/rte_eal.h
@@ -61,6 +61,7 @@ extern "C" {
 enum rte_lcore_role_t {
 	ROLE_RTE,
 	ROLE_OFF,
+	ROLE_SERVICE,
 };
 
 /**
@@ -80,6 +81,7 @@ enum rte_proc_type_t {
 struct rte_config {
 	uint32_t master_lcore;       /**< Id of the master lcore */
 	uint32_t lcore_count;        /**< Number of available logical cores. */
+	uint32_t service_lcore_count;/**< Number of available service cores. */
 	enum rte_lcore_role_t lcore_role[RTE_MAX_LCORE]; /**< State of cores. */
 
 	/** Primary or secondary configuration */
@@ -185,6 +187,8 @@ int rte_eal_iopl_init(void);
  *
  *     EPROTO indicates that the PCI bus is either not present, or is not
  *            readable by the eal.
+ *
+ *     ENOEXEC indicates that a service core failed to launch successfully.
  */
 int rte_eal_init(int argc, char **argv);
 
diff --git a/lib/librte_eal/common/include/rte_lcore.h b/lib/librte_eal/common/include/rte_lcore.h
index fe7b586..50e0d0f 100644
--- a/lib/librte_eal/common/include/rte_lcore.h
+++ b/lib/librte_eal/common/include/rte_lcore.h
@@ -73,6 +73,7 @@ struct lcore_config {
 	unsigned core_id;          /**< core number on socket for this lcore */
 	int core_index;            /**< relative index, starting from 0 */
 	rte_cpuset_t cpuset;       /**< cpu set which the lcore affinity to */
+	uint8_t core_role;         /**< role of core eg: OFF, RTE, SERVICE */
 };
 
 /**
@@ -175,7 +176,7 @@ rte_lcore_is_enabled(unsigned lcore_id)
 	struct rte_config *cfg = rte_eal_get_configuration();
 	if (lcore_id >= RTE_MAX_LCORE)
 		return 0;
-	return cfg->lcore_role[lcore_id] != ROLE_OFF;
+	return cfg->lcore_role[lcore_id] == ROLE_RTE;
 }
 
 /**
diff --git a/lib/librte_eal/common/include/rte_service.h b/lib/librte_eal/common/include/rte_service.h
new file mode 100644
index 0000000..7c6f738
--- /dev/null
+++ b/lib/librte_eal/common/include/rte_service.h
@@ -0,0 +1,387 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_SERVICE_H_
+#define _RTE_SERVICE_H_
+
+/**
+ * @file
+ *
+ * Service functions
+ *
+ * The service functionality provided by this header allows a DPDK component
+ * to indicate that it requires a function call in order for it to perform
+ * its processing.
+ *
+ * An example usage of this functionality would be a component that registers
+ * a service to perform a particular packet processing duty: for example the
+ * eventdev software PMD. At startup the application requests all services
+ * that have been registered, and the cores in the service-coremask run the
+ * required services. The EAL removes these number of cores from the available
+ * runtime cores, and dedicates them to performing service-core workloads. The
+ * application has access to the remaining lcores as normal.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include<stdio.h>
+#include <stdint.h>
+#include <sys/queue.h>
+
+#include <rte_lcore.h>
+
+/* forward declaration only. Definition in rte_service_private.h */
+struct rte_service_spec;
+
+#define RTE_SERVICE_NAME_MAX 32
+
+/* Capabilities of a service.
+ *
+ * Use the *rte_service_probe_capability* function to check if a service is
+ * capable of a specific capability.
+ */
+/** When set, the service is capable of having multiple threads run it at the
+ *  same time.
+ */
+#define RTE_SERVICE_CAP_MT_SAFE (1 << 0)
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ *  Return the number of services registered.
+ *
+ * The number of services registered can be passed to *rte_service_get_by_id*,
+ * enabling the application to retrieve the specification of each service.
+ *
+ * @return The number of services registered.
+ */
+uint32_t rte_service_get_count(void);
+
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Return the specification of a service by integer id.
+ *
+ * This function provides the specification of a service. This can be used by
+ * the application to understand what the service represents. The service
+ * must not be modified by the application directly, only passed to the various
+ * rte_service_* functions.
+ *
+ * @param id The integer id of the service to retrieve
+ * @retval non-zero A valid pointer to the service_spec
+ * @retval NULL Invalid *id* provided.
+ */
+struct rte_service_spec *rte_service_get_by_id(uint32_t id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Return the specification of a service by name.
+ *
+ * This function provides the specification of a service using the service name
+ * as lookup key. This can be used by the application to understand what the
+ * service represents. The service must not be modified by the application
+ * directly, only passed to the various rte_service_* functions.
+ *
+ * @param name The name of the service to retrieve
+ * @retval non-zero A valid pointer to the service_spec
+ * @retval NULL Invalid *name* provided.
+ */
+struct rte_service_spec *rte_service_get_by_name(const char *name);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Return the name of the service.
+ *
+ * @return A pointer to the name of the service. The returned pointer remains
+ *         in ownership of the service, and the application must not free it.
+ */
+const char *rte_service_get_name(const struct rte_service_spec *service);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Check if a service has a specific capability.
+ *
+ * This function returns if *service* has implements *capability*.
+ * See RTE_SERVICE_CAP_* defines for a list of valid capabilities.
+ * @retval 1 Capability supported by this service instance
+ * @retval 0 Capability not supported by this service instance
+ */
+int32_t rte_service_probe_capability(const struct rte_service_spec *service,
+				     uint32_t capability);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Enable a core to run a service.
+ *
+ * Each core can be added or removed from running specific services. This
+ * functions adds *lcore* to the set of cores that will run *service*.
+ *
+ * If multiple cores are enabled on a service, an atomic is used to ensure that
+ * only one cores runs the service at a time. The exception to this is when
+ * a service indicates that it is multi-thread safe by setting the capability
+ * called RTE_SERVICE_CAP_MT_SAFE. With the multi-thread safe capability set,
+ * the service function can be run on multiple threads at the same time.
+ *
+ * @retval 0 lcore added successfully
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_enable_on_lcore(struct rte_service_spec *service,
+				   uint32_t lcore);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Disable a core to run a service.
+ *
+ * Each core can be added or removed from running specific services. This
+ * functions removes *lcore* to the set of cores that will run *service*.
+ *
+ * @retval 0 Lcore removed successfully
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_disable_on_lcore(struct rte_service_spec *service,
+				   uint32_t lcore);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Return if an lcore is enabled for the service.
+ *
+ * This function allows the application to query if *lcore* is currently set to
+ * run *service*.
+ *
+ * @retval 1 Lcore enabled on this lcore
+ * @retval 0 Lcore disabled on this lcore
+ * @retval -EINVAL An invalid service or lcore was provided.
+ */
+int32_t rte_service_get_enabled_on_lcore(struct rte_service_spec *service,
+					uint32_t lcore);
+
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Enable *service* to run.
+ *
+ * This function switches on a service during runtime.
+ * @retval 0 The service was successfully started
+ */
+int32_t rte_service_start(struct rte_service_spec *service);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Disable *service*.
+ *
+ * Switch off a service, so it is not run until it is *rte_service_start* is
+ * called on it.
+ * @retval 0 Service successfully switched off
+ */
+int32_t rte_service_stop(struct rte_service_spec *service);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Returns if *service* is currently running.
+ *
+ * This function returns true if the service has been started using
+ * *rte_service_start*, AND a service core is mapped to the service. This
+ * function can be used to ensure that the service will be run.
+ *
+ * @retval 1 Service is currently running, and has a service lcore mapped
+ * @retval 0 Service is currently stopped, or no service lcore is mapped
+ * @retval -EINVAL Invalid service pointer provided
+ */
+int32_t rte_service_is_running(const struct rte_service_spec *service);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Start a service core.
+ *
+ * Starting a core makes the core begin polling. Any services assigned to it
+ * will be run as fast as possible.
+ *
+ * @retval 0 Success
+ * @retval -EINVAL Failed to start core. The *lcore_id* passed in is not
+ *          currently assigned to be a service core.
+ */
+int32_t rte_service_lcore_start(uint32_t lcore_id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Stop a service core.
+ *
+ * Stopping a core makes the core become idle, but remains  assigned as a
+ * service core.
+ *
+ * @retval 0 Success
+ * @retval -EINVAL Invalid *lcore_id* provided
+ * @retval -EALREADY Already stopped core
+ * @retval -EBUSY Failed to stop core, as it would cause a service to not
+ *          be run, as this is the only core currently running the service.
+ *          The application must stop the service first, and then stop the
+ *          lcore.
+ */
+int32_t rte_service_lcore_stop(uint32_t lcore_id);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Adds lcore to the list of service cores.
+ *
+ * This functions can be used at runtime in order to modify the service core
+ * mask.
+ *
+ * @retval 0 Success
+ * @retval -EBUSY lcore is busy, and not available for service core duty
+ * @retval -EALREADY lcore is already added to the service core list
+ * @retval -EINVAL Invalid lcore provided
+ */
+int32_t rte_service_lcore_add(uint32_t lcore);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Removes lcore from the list of service cores.
+ *
+ * This can fail if the core is not stopped, see *rte_service_core_stop*.
+ *
+ * @retval 0 Success
+ * @retval -EBUSY Lcore is not stopped, stop service core before removing.
+ * @retval -EINVAL failed to add lcore to service core mask.
+ */
+int32_t rte_service_lcore_del(uint32_t lcore);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Retrieve the number of service cores currently available.
+ *
+ * This function returns the integer count of service cores available. The
+ * service core count can be used in mapping logic when creating mappings
+ * from service cores to services.
+ *
+ * See *rte_service_lcore_list* for details on retrieving the lcore_id of each
+ * service core.
+ *
+ * @return The number of service cores currently configured.
+ */
+int32_t rte_service_lcore_count(void);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Resets all service core mappings. This does not remove the service cores
+ * from duty, just unmaps all services / cores, and stops() the service cores.
+ * The runstate of services is not modified.
+ *
+ * @retval 0 Success
+ */
+int32_t rte_service_lcore_reset_all(void);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Enable or disable statistics collection for *service*.
+ *
+ * This function enables per core, per-service cycle count collection.
+ * @param service The service to enable statistics gathering on.
+ * @param enable Zero to disable statistics, non-zero to enable.
+ * @retval 0 Success
+ * @retval -EINVAL Invalid service pointer passed
+ */
+int32_t rte_service_set_stats_enable(struct rte_service_spec *service,
+				  int32_t enable);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Retrieve the list of currently enabled service cores.
+ *
+ * This function fills in an application supplied array, with each element
+ * indicating the lcore_id of a service core.
+ *
+ * Adding and removing service cores can be performed using
+ * *rte_service_lcore_add* and *rte_service_lcore_del*.
+ * @param [out] array An array of at least *rte_service_lcore_count* items.
+ *              If statically allocating the buffer, use RTE_MAX_LCORE.
+ * @param [out] n The size of *array*.
+ * @retval >=0 Number of service cores that have been populated in the array
+ * @retval -ENOMEM The provided array is not large enough to fill in the
+ *          service core list. No items have been populated, call this function
+ *          with a size of at least *rte_service_core_count* items.
+ */
+int32_t rte_service_lcore_list(uint32_t array[], uint32_t n);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Dumps any information available about the service. If service is NULL,
+ * dumps info for all services.
+ */
+int32_t rte_service_dump(FILE *f, struct rte_service_spec *service);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif /* _RTE_SERVICE_H_ */
diff --git a/lib/librte_eal/common/include/rte_service_component.h b/lib/librte_eal/common/include/rte_service_component.h
new file mode 100644
index 0000000..7a946a1
--- /dev/null
+++ b/lib/librte_eal/common/include/rte_service_component.h
@@ -0,0 +1,144 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_SERVICE_PRIVATE_H_
+#define _RTE_SERVICE_PRIVATE_H_
+
+/* This file specifies the internal service specification.
+ * Include this file if you are writing a component that requires CPU cycles to
+ * operate, and you wish to run the component using service cores
+ */
+
+#include <rte_service.h>
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Signature of callback function to run a service.
+ */
+typedef int32_t (*rte_service_func)(void *args);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * The specification of a service.
+ *
+ * This struct contains metadata about the service itself, the callback
+ * function to run one iteration of the service, a userdata pointer, flags etc.
+ */
+struct rte_service_spec {
+	/** The name of the service. This should be used by the application to
+	 * understand what purpose this service provides.
+	 */
+	char name[RTE_SERVICE_NAME_MAX];
+	/** The callback to invoke to run one iteration of the service. */
+	rte_service_func callback;
+	/** The userdata pointer provided to the service callback. */
+	void *callback_userdata;
+	/** Flags to indicate the capabilities of this service. See defines in
+	 * the public header file for values of RTE_SERVICE_CAP_*
+	 */
+	uint32_t capabilities;
+	/** NUMA socket ID that this service is affinitized to */
+	int socket_id;
+};
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Register a new service.
+ *
+ * A service represents a component that the requires CPU time periodically to
+ * achieve its purpose.
+ *
+ * For example the eventdev SW PMD requires CPU cycles to perform its
+ * scheduling. This can be achieved by registering it as a service, and the
+ * application can then assign CPU resources to it using
+ * *rte_service_set_coremask*.
+ *
+ * @param spec The specification of the service to register
+ * @retval 0 Successfully registered the service.
+ *         -EINVAL Attempted to register an invalid service (eg, no callback
+ *         set)
+ */
+int32_t rte_service_register(const struct rte_service_spec *spec);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Unregister a service.
+ *
+ * The service being removed must be stopped before calling this function.
+ *
+ * @retval 0 The service was successfully unregistered.
+ * @retval -EBUSY The service is currently running, stop the service before
+ *          calling unregister. No action has been taken.
+ */
+int32_t rte_service_unregister(struct rte_service_spec *service);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Private function to allow EAL to initialized default mappings.
+ *
+ * This function iterates all the services, and maps then to the available
+ * cores. Based on the capabilities of the services, they are set to run on the
+ * available cores in a round-robin manner.
+ *
+ * @retval 0 Success
+ * @retval -ENOTSUP No service lcores in use
+ * @retval -EINVAL Error while iterating over services
+ * @retval -ENODEV Error in enabling service lcore on a service
+ * @retval -ENOEXEC Error when starting services
+ */
+int32_t rte_service_start_with_defaults(void);
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Initialize the service library.
+ *
+ * In order to use the service library, it must be initialized. EAL initializes
+ * the library at startup.
+ *
+ * @retval 0 Success
+ * @retval -EALREADY Service library is already initialized
+ */
+int32_t rte_service_init(void);
+
+#endif /* _RTE_SERVICE_PRIVATE_H_ */
diff --git a/lib/librte_eal/common/rte_service.c b/lib/librte_eal/common/rte_service.c
new file mode 100644
index 0000000..e82b9ad
--- /dev/null
+++ b/lib/librte_eal/common/rte_service.c
@@ -0,0 +1,704 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *   All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <limits.h>
+#include <string.h>
+#include <dirent.h>
+
+#include <rte_service.h>
+#include "include/rte_service_component.h"
+
+#include <rte_eal.h>
+#include <rte_lcore.h>
+#include <rte_common.h>
+#include <rte_debug.h>
+#include <rte_cycles.h>
+#include <rte_atomic.h>
+#include <rte_memory.h>
+#include <rte_malloc.h>
+
+#define RTE_SERVICE_NUM_MAX 64
+
+#define SERVICE_F_REGISTERED    (1 << 0)
+#define SERVICE_F_STATS_ENABLED (1 << 1)
+
+/* runstates for services and lcores, denoting if they are active or not */
+#define RUNSTATE_STOPPED 0
+#define RUNSTATE_RUNNING 1
+
+/* internal representation of a service */
+struct rte_service_spec_impl {
+	/* public part of the struct */
+	struct rte_service_spec spec;
+
+	/* atomic lock that when set indicates a service core is currently
+	 * running this service callback. When not set, a core may take the
+	 * lock and then run the service callback.
+	 */
+	rte_atomic32_t execute_lock;
+
+	/* API set/get-able variables */
+	int32_t runstate;
+	uint8_t internal_flags;
+
+	/* per service statistics */
+	uint32_t num_mapped_cores;
+	uint64_t calls;
+	uint64_t cycles_spent;
+} __rte_cache_aligned;
+
+/* the internal values of a service core */
+struct core_state {
+	/* map of services IDs are run on this core */
+	uint64_t service_mask;
+	uint8_t runstate; /* running or stopped */
+	uint8_t is_service_core; /* set if core is currently a service core */
+
+	/* extreme statistics */
+	uint64_t calls_per_service[RTE_SERVICE_NUM_MAX];
+} __rte_cache_aligned;
+
+static uint32_t rte_service_count;
+static struct rte_service_spec_impl *rte_services;
+static struct core_state *lcore_states;
+static uint32_t rte_service_library_initialized;
+
+int32_t rte_service_init(void)
+{
+	if (rte_service_library_initialized) {
+		printf("service library init() called, init flag %d\n",
+			rte_service_library_initialized);
+		return -EALREADY;
+	}
+
+	rte_services = rte_calloc("rte_services", RTE_SERVICE_NUM_MAX,
+			sizeof(struct rte_service_spec_impl),
+			RTE_CACHE_LINE_SIZE);
+	if (!rte_services) {
+		printf("error allocating rte services array\n");
+		return -ENOMEM;
+	}
+
+	lcore_states = rte_calloc("rte_service_core_states", RTE_MAX_LCORE,
+			sizeof(struct core_state), RTE_CACHE_LINE_SIZE);
+	if (!lcore_states) {
+		printf("error allocating core states array\n");
+		return -ENOMEM;
+	}
+
+	int i;
+	int count = 0;
+	struct rte_config *cfg = rte_eal_get_configuration();
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		if (lcore_config[i].core_role == ROLE_SERVICE) {
+			if ((unsigned int)i == cfg->master_lcore)
+				continue;
+			rte_service_lcore_add(i);
+			count++;
+		}
+	}
+
+	rte_service_library_initialized = 1;
+	return 0;
+}
+
+/* returns 1 if service is registered and has not been unregistered
+ * Returns 0 if service never registered, or has been unregistered
+ */
+static inline int
+service_valid(uint32_t id)
+{
+	return !!(rte_services[id].internal_flags & SERVICE_F_REGISTERED);
+}
+
+/* returns 1 if statistics should be colleced for service
+ * Returns 0 if statistics should not be collected for service
+ */
+static inline int
+service_stats_enabled(struct rte_service_spec_impl *impl)
+{
+	return !!(impl->internal_flags & SERVICE_F_STATS_ENABLED);
+}
+
+static inline int
+service_mt_safe(struct rte_service_spec_impl *s)
+{
+	return s->spec.capabilities & RTE_SERVICE_CAP_MT_SAFE;
+}
+
+int32_t rte_service_set_stats_enable(struct rte_service_spec *service,
+				  int32_t enabled)
+{
+	struct rte_service_spec_impl *impl =
+		(struct rte_service_spec_impl *)service;
+	if (!impl)
+		return -EINVAL;
+
+	if (enabled)
+		impl->internal_flags |= SERVICE_F_STATS_ENABLED;
+	else
+		impl->internal_flags &= ~(SERVICE_F_STATS_ENABLED);
+
+	return 0;
+}
+
+uint32_t
+rte_service_get_count(void)
+{
+	return rte_service_count;
+}
+
+struct rte_service_spec *
+rte_service_get_by_id(uint32_t id)
+{
+	struct rte_service_spec *service = NULL;
+	if (id < rte_service_count)
+		service = (struct rte_service_spec *)&rte_services[id];
+
+	return service;
+}
+
+struct rte_service_spec *rte_service_get_by_name(const char *name)
+{
+	struct rte_service_spec *service = NULL;
+	int i;
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (service_valid(i) &&
+				strcmp(name, rte_services[i].spec.name) == 0) {
+			service = (struct rte_service_spec *)&rte_services[i];
+			break;
+		}
+	}
+
+	return service;
+}
+
+const char *
+rte_service_get_name(const struct rte_service_spec *service)
+{
+	return service->name;
+}
+
+int32_t
+rte_service_probe_capability(const struct rte_service_spec *service,
+			     uint32_t capability)
+{
+	return service->capabilities & capability;
+}
+
+int32_t
+rte_service_is_running(const struct rte_service_spec *spec)
+{
+	const struct rte_service_spec_impl *impl =
+		(const struct rte_service_spec_impl *)spec;
+	if (!impl)
+		return -EINVAL;
+
+	return (impl->runstate == RUNSTATE_RUNNING) &&
+		(impl->num_mapped_cores > 0);
+}
+
+int32_t
+rte_service_register(const struct rte_service_spec *spec)
+{
+	uint32_t i;
+	int32_t free_slot = -1;
+
+	if (spec->callback == NULL || strlen(spec->name) == 0)
+		return -EINVAL;
+
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (!service_valid(i)) {
+			free_slot = i;
+			break;
+		}
+	}
+
+	if ((free_slot < 0) || (i == RTE_SERVICE_NUM_MAX))
+		return -ENOSPC;
+
+	struct rte_service_spec_impl *s = &rte_services[free_slot];
+	s->spec = *spec;
+	s->internal_flags |= SERVICE_F_REGISTERED;
+
+	rte_smp_wmb();
+	rte_service_count++;
+
+	return 0;
+}
+
+int32_t
+rte_service_unregister(struct rte_service_spec *spec)
+{
+	struct rte_service_spec_impl *s = NULL;
+	struct rte_service_spec_impl *spec_impl =
+		(struct rte_service_spec_impl *)spec;
+
+	uint32_t i;
+	uint32_t service_id;
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (&rte_services[i] == spec_impl) {
+			s = spec_impl;
+			service_id = i;
+			break;
+		}
+	}
+
+	if (!s)
+		return -EINVAL;
+
+	rte_service_count--;
+	rte_smp_wmb();
+
+	s->internal_flags &= ~(SERVICE_F_REGISTERED);
+
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		lcore_states[i].service_mask &= ~(1 << service_id);
+
+	memset(&rte_services[service_id], 0,
+			sizeof(struct rte_service_spec_impl));
+
+	return 0;
+}
+
+int32_t
+rte_service_start(struct rte_service_spec *service)
+{
+	struct rte_service_spec_impl *s =
+		(struct rte_service_spec_impl *)service;
+	s->runstate = RUNSTATE_RUNNING;
+	rte_smp_wmb();
+	return 0;
+}
+
+int32_t
+rte_service_stop(struct rte_service_spec *service)
+{
+	struct rte_service_spec_impl *s =
+		(struct rte_service_spec_impl *)service;
+	s->runstate = RUNSTATE_STOPPED;
+	rte_smp_wmb();
+	return 0;
+}
+
+static int32_t
+rte_service_runner_func(void *arg)
+{
+	RTE_SET_USED(arg);
+	uint32_t i;
+	const int lcore = rte_lcore_id();
+	struct core_state *cs = &lcore_states[lcore];
+
+	while (lcore_states[lcore].runstate == RUNSTATE_RUNNING) {
+		const uint64_t service_mask = cs->service_mask;
+		for (i = 0; i < rte_service_count; i++) {
+			struct rte_service_spec_impl *s = &rte_services[i];
+			if (s->runstate != RUNSTATE_RUNNING ||
+					!(service_mask & (1 << i)))
+				continue;
+
+			/* check do we need cmpset, if MT safe or <= 1 core
+			 * mapped, atomic ops are not required.
+			 */
+			const int need_cmpset = !((service_mt_safe(s) == 0) &&
+						(s->num_mapped_cores > 1));
+			uint32_t *lock = (uint32_t *)&s->execute_lock;
+
+			if (need_cmpset || rte_atomic32_cmpset(lock, 0, 1)) {
+				void *userdata = s->spec.callback_userdata;
+
+				if (service_stats_enabled(s)) {
+					uint64_t start = rte_rdtsc();
+					s->spec.callback(userdata);
+					uint64_t end = rte_rdtsc();
+					s->cycles_spent += end - start;
+					cs->calls_per_service[i]++;
+					s->calls++;
+				} else
+					s->spec.callback(userdata);
+
+				if (need_cmpset)
+					rte_atomic32_clear(&s->execute_lock);
+			}
+		}
+
+		rte_smp_rmb();
+	}
+
+	lcore_config[lcore].state = WAIT;
+
+	return 0;
+}
+
+int32_t
+rte_service_lcore_count(void)
+{
+	int32_t count = 0;
+	uint32_t i;
+	for (i = 0; i < RTE_MAX_LCORE; i++)
+		count += lcore_states[i].is_service_core;
+	return count;
+}
+
+int32_t
+rte_service_lcore_list(uint32_t array[], uint32_t n)
+{
+	uint32_t count = rte_service_lcore_count();
+	if (count > n)
+		return -ENOMEM;
+
+	if (!array)
+		return -EINVAL;
+
+	uint32_t i;
+	uint32_t idx = 0;
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct core_state *cs = &lcore_states[i];
+		if (cs->is_service_core) {
+			array[idx] = i;
+			idx++;
+		}
+	}
+
+	return count;
+}
+
+int32_t
+rte_service_start_with_defaults(void)
+{
+	/* create a default mapping from cores to services, then start the
+	 * services to make them transparent to unaware applications.
+	 */
+	uint32_t i;
+	int ret;
+	uint32_t count = rte_service_get_count();
+
+	int32_t lcore_iter = 0;
+	uint32_t ids[RTE_MAX_LCORE];
+	int32_t lcore_count = rte_service_lcore_list(ids, RTE_MAX_LCORE);
+
+	if (lcore_count == 0)
+		return -ENOTSUP;
+
+	for (i = 0; (int)i < lcore_count; i++)
+		rte_service_lcore_start(ids[i]);
+
+	for (i = 0; i < count; i++) {
+		struct rte_service_spec *s = rte_service_get_by_id(i);
+		if (!s)
+			return -EINVAL;
+
+		/* do 1:1 core mapping here, with each service getting
+		 * assigned a single core by default. Adding multiple services
+		 * should multiplex to a single core, or 1:1 if there are the
+		 * same amount of services as service-cores
+		 */
+		ret = rte_service_enable_on_lcore(s, ids[lcore_iter]);
+		if (ret)
+			return -ENODEV;
+
+		lcore_iter++;
+		if (lcore_iter >= lcore_count)
+			lcore_iter = 0;
+
+		ret = rte_service_start(s);
+		if (ret)
+			return -ENOEXEC;
+	}
+
+	return 0;
+}
+
+static int32_t
+service_update(struct rte_service_spec *service, uint32_t lcore,
+		uint32_t *set, uint32_t *enabled)
+{
+	uint32_t i;
+	int32_t sid = -1;
+
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if ((struct rte_service_spec *)&rte_services[i] == service &&
+				service_valid(i)) {
+			sid = i;
+			break;
+		}
+	}
+
+	if (sid == -1 || lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	if (!lcore_states[lcore].is_service_core)
+		return -EINVAL;
+
+	if (set) {
+		if (*set) {
+			lcore_states[lcore].service_mask |=  (1 << sid);
+			rte_services[sid].num_mapped_cores++;
+		} else {
+			lcore_states[lcore].service_mask &= ~(1 << sid);
+			rte_services[sid].num_mapped_cores--;
+		}
+	}
+
+	if (enabled)
+		*enabled = (lcore_states[lcore].service_mask & (1 << sid));
+
+	rte_smp_wmb();
+
+	return 0;
+}
+
+int32_t rte_service_get_enabled_on_lcore(struct rte_service_spec *service,
+					uint32_t lcore)
+{
+	uint32_t enabled;
+	int ret = service_update(service, lcore, 0, &enabled);
+	if (ret == 0)
+		return enabled;
+	return -EINVAL;
+}
+
+int32_t
+rte_service_enable_on_lcore(struct rte_service_spec *service, uint32_t lcore)
+{
+	uint32_t on = 1;
+	return service_update(service, lcore, &on, 0);
+}
+
+int32_t
+rte_service_disable_on_lcore(struct rte_service_spec *service, uint32_t lcore)
+{
+	uint32_t off = 0;
+	return service_update(service, lcore, &off, 0);
+}
+
+int32_t rte_service_lcore_reset_all(void)
+{
+	/* loop over cores, reset all to mask 0 */
+	uint32_t i;
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		lcore_states[i].service_mask = 0;
+		lcore_states[i].is_service_core = 0;
+		lcore_states[i].runstate = RUNSTATE_STOPPED;
+	}
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++)
+		rte_services[i].num_mapped_cores = 0;
+
+	rte_smp_wmb();
+
+	return 0;
+}
+
+static void
+set_lcore_state(uint32_t lcore, int32_t state)
+{
+	/* mark core state in hugepage backed config */
+	struct rte_config *cfg = rte_eal_get_configuration();
+	cfg->lcore_role[lcore] = state;
+
+	/* mark state in process local lcore_config */
+	lcore_config[lcore].core_role = state;
+
+	/* update per-lcore optimized state tracking */
+	lcore_states[lcore].is_service_core = (state == ROLE_SERVICE);
+}
+
+int32_t
+rte_service_lcore_add(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+	if (lcore_states[lcore].is_service_core)
+		return -EALREADY;
+
+	set_lcore_state(lcore, ROLE_SERVICE);
+
+	/* ensure that after adding a core the mask and state are defaults */
+	lcore_states[lcore].service_mask = 0;
+	lcore_states[lcore].runstate = RUNSTATE_STOPPED;
+
+	rte_smp_wmb();
+	return 0;
+}
+
+int32_t
+rte_service_lcore_del(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	struct core_state *cs = &lcore_states[lcore];
+	if (!cs->is_service_core)
+		return -EINVAL;
+
+	if (cs->runstate != RUNSTATE_STOPPED)
+		return -EBUSY;
+
+	set_lcore_state(lcore, ROLE_RTE);
+
+	rte_smp_wmb();
+	return 0;
+}
+
+int32_t
+rte_service_lcore_start(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	struct core_state *cs = &lcore_states[lcore];
+	if (!cs->is_service_core)
+		return -EINVAL;
+
+	if (cs->runstate == RUNSTATE_RUNNING)
+		return -EALREADY;
+
+	/* set core to run state first, and then launch otherwise it will
+	 * return immediately as runstate keeps it in the service poll loop
+	 */
+	lcore_states[lcore].runstate = RUNSTATE_RUNNING;
+
+	int ret = rte_eal_remote_launch(rte_service_runner_func, 0, lcore);
+	/* returns -EBUSY if the core is already launched, 0 on success */
+	return ret;
+}
+
+int32_t
+rte_service_lcore_stop(uint32_t lcore)
+{
+	if (lcore >= RTE_MAX_LCORE)
+		return -EINVAL;
+
+	if (lcore_states[lcore].runstate == RUNSTATE_STOPPED)
+		return -EALREADY;
+
+	uint32_t i;
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		int32_t enabled = lcore_states[i].service_mask & (1 << i);
+		int32_t service_running = rte_services[i].runstate !=
+						RUNSTATE_STOPPED;
+		int32_t only_core = rte_services[i].num_mapped_cores == 1;
+
+		/* if the core is mapped, and the service is running, and this
+		 * is the only core that is mapped, the service would cease to
+		 * run if this core stopped, so fail instead.
+		 */
+		if (enabled && service_running && only_core)
+			return -EBUSY;
+	}
+
+	lcore_states[lcore].runstate = RUNSTATE_STOPPED;
+
+	return 0;
+}
+
+static void
+rte_service_dump_one(FILE *f, struct rte_service_spec_impl *s,
+		     uint64_t all_cycles, uint32_t reset)
+{
+	/* avoid divide by zero */
+	if (all_cycles == 0)
+		all_cycles = 1;
+
+	int calls = 1;
+	if (s->calls != 0)
+		calls = s->calls;
+
+	fprintf(f, "  %s: stats %d\tcalls %"PRIu64"\tcycles %"
+			PRIu64"\tavg: %"PRIu64"\n",
+			s->spec.name, service_stats_enabled(s), s->calls,
+			s->cycles_spent, s->cycles_spent / calls);
+
+	if (reset) {
+		s->cycles_spent = 0;
+		s->calls = 0;
+	}
+}
+
+static void
+service_dump_calls_per_lcore(FILE *f, uint32_t lcore, uint32_t reset)
+{
+	uint32_t i;
+	struct core_state *cs = &lcore_states[lcore];
+
+	fprintf(f, "%02d\t", lcore);
+	for (i = 0; i < RTE_SERVICE_NUM_MAX; i++) {
+		if (!service_valid(i))
+			continue;
+		fprintf(f, "%"PRIu64"\t", cs->calls_per_service[i]);
+		if (reset)
+			cs->calls_per_service[i] = 0;
+	}
+	fprintf(f, "\n");
+}
+
+int32_t rte_service_dump(FILE *f, struct rte_service_spec *service)
+{
+	uint32_t i;
+
+	uint64_t total_cycles = 0;
+	for (i = 0; i < rte_service_count; i++) {
+		if (!service_valid(i))
+			continue;
+		total_cycles += rte_services[i].cycles_spent;
+	}
+
+	if (service) {
+		struct rte_service_spec_impl *s =
+			(struct rte_service_spec_impl *)service;
+		fprintf(f, "Service %s Summary\n", s->spec.name);
+		uint32_t reset = 0;
+		rte_service_dump_one(f, s, total_cycles, reset);
+		return 0;
+	}
+
+	fprintf(f, "Services Summary\n");
+	for (i = 0; i < rte_service_count; i++) {
+		uint32_t reset = 1;
+		rte_service_dump_one(f, &rte_services[i], total_cycles, reset);
+	}
+
+	fprintf(f, "Service Cores Summary\n");
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		if (lcore_config[i].core_role != ROLE_SERVICE)
+			continue;
+
+		uint32_t reset = 0;
+		service_dump_calls_per_lcore(f, i, reset);
+	}
+
+	return 0;
+}
diff --git a/lib/librte_eal/linuxapp/eal/Makefile b/lib/librte_eal/linuxapp/eal/Makefile
index 8651e27..e6ab6c3 100644
--- a/lib/librte_eal/linuxapp/eal/Makefile
+++ b/lib/librte_eal/linuxapp/eal/Makefile
@@ -99,6 +99,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_malloc.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_elem.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_heap.c
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_keepalive.c
+SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_service.c
 
 # from arch dir
 SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_cpuflags.c
diff --git a/lib/librte_eal/linuxapp/eal/eal_thread.c b/lib/librte_eal/linuxapp/eal/eal_thread.c
index 9f88530..831ba07 100644
--- a/lib/librte_eal/linuxapp/eal/eal_thread.c
+++ b/lib/librte_eal/linuxapp/eal/eal_thread.c
@@ -184,7 +184,14 @@ eal_thread_loop(__attribute__((unused)) void *arg)
 		ret = lcore_config[lcore_id].f(fct_arg);
 		lcore_config[lcore_id].ret = ret;
 		rte_wmb();
-		lcore_config[lcore_id].state = FINISHED;
+
+		/* when a service core returns, it should go directly to WAIT
+		 * state, because the application will not lcore_wait() for it.
+		 */
+		if (lcore_config[lcore_id].core_role == ROLE_SERVICE)
+			lcore_config[lcore_id].state = WAIT;
+		else
+			lcore_config[lcore_id].state = FINISHED;
 	}
 
 	/* never reached */
diff --git a/lib/librte_eal/linuxapp/eal/rte_eal_version.map b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
index 0f9e009..fbaec39 100644
--- a/lib/librte_eal/linuxapp/eal/rte_eal_version.map
+++ b/lib/librte_eal/linuxapp/eal/rte_eal_version.map
@@ -214,5 +214,28 @@ EXPERIMENTAL {
 	rte_eal_devargs_parse;
 	rte_eal_hotplug_add;
 	rte_eal_hotplug_remove;
+	rte_service_disable_on_lcore;
+	rte_service_dump;
+	rte_service_enable_on_lcore;
+	rte_service_get_by_id;
+	rte_service_get_by_name;
+	rte_service_get_count;
+	rte_service_get_enabled_on_lcore;
+	rte_service_is_running;
+	rte_service_lcore_add;
+	rte_service_lcore_count;
+	rte_service_lcore_del;
+	rte_service_lcore_list;
+	rte_service_lcore_reset_all;
+	rte_service_lcore_start;
+	rte_service_lcore_stop;
+	rte_service_probe_capability;
+	rte_service_register;
+	rte_service_reset;
+	rte_service_set_stats_enable;
+	rte_service_start;
+	rte_service_start_with_defaults;
+	rte_service_stop;
+	rte_service_unregister;
 
 } DPDK_17.08;
-- 
2.7.4

[PATCH v5 2/7] service cores: EAL init changes

[Harry van Haaren]

This commit shows the changes required in rte_eal_init()
to transparently launch the service threads. The threads
are launched into the service worker functions here because
after rte_eal_init() the application is not gauranteed to
call any other DPDK API.

As the registration of services happens at initialization
time, the services that require CPU time are already available
when we reach the end of rte_eal_init().

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

---

v5:
- Simplify EAL code, calling rte_service_start_with_defaults() (Jerin)
- Added Ack from ML

v4:
- Added #include for service cores in BSD eal.c

v2 comments:
- Include BSD implementation (Jerin)
- Move details of core-tracking into rte_service_lcore_add(Jerin)
- Given there are changes other to suggested, not using Ack
---
 lib/librte_eal/bsdapp/eal/eal.c   | 18 ++++++++++++++++++
 lib/librte_eal/linuxapp/eal/eal.c | 18 ++++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c
index 05f0c1f..3e7a3a8 100644
--- a/lib/librte_eal/bsdapp/eal/eal.c
+++ b/lib/librte_eal/bsdapp/eal/eal.c
@@ -72,6 +72,7 @@
 #include <rte_common.h>
 #include <rte_version.h>
 #include <rte_atomic.h>
+#include <rte_service_component.h>
 #include <malloc_heap.h>
 
 #include "eal_private.h"
@@ -653,6 +654,14 @@ rte_eal_init(int argc, char **argv)
 	rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER);
 	rte_eal_mp_wait_lcore();
 
+	/* initialize services so vdevs register service during bus_probe. */
+	ret = rte_service_init();
+	if (ret) {
+		rte_eal_init_alert("rte_service_init() failed\n");
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	/* Probe all the buses and devices/drivers on them */
 	if (rte_bus_probe()) {
 		rte_eal_init_alert("Cannot probe devices\n");
@@ -660,6 +669,15 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 
+	/* initialize default service/lcore mappings and start running. Ignore
+	 * -ENOTSUP, as it indicates no service coremask passed to EAL.
+	 */
+	ret = rte_service_start_with_defaults();
+	if (ret < 0 && ret != -ENOTSUP) {
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	rte_eal_mcfg_complete();
 
 	return fctret;
diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
index 7c78f2d..2914646 100644
--- a/lib/librte_eal/linuxapp/eal/eal.c
+++ b/lib/librte_eal/linuxapp/eal/eal.c
@@ -78,6 +78,7 @@
 #include <rte_version.h>
 #include <rte_atomic.h>
 #include <malloc_heap.h>
+#include <rte_service_component.h>
 
 #include "eal_private.h"
 #include "eal_thread.h"
@@ -932,6 +933,14 @@ rte_eal_init(int argc, char **argv)
 	rte_eal_mp_remote_launch(sync_func, NULL, SKIP_MASTER);
 	rte_eal_mp_wait_lcore();
 
+	/* initialize services so vdevs register service during bus_probe. */
+	ret = rte_service_init();
+	if (ret) {
+		rte_eal_init_alert("rte_service_init() failed\n");
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	/* Probe all the buses and devices/drivers on them */
 	if (rte_bus_probe()) {
 		rte_eal_init_alert("Cannot probe devices\n");
@@ -939,6 +948,15 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 
+	/* initialize default service/lcore mappings and start running. Ignore
+	 * -ENOTSUP, as it indicates no service coremask passed to EAL.
+	 */
+	ret = rte_service_start_with_defaults();
+	if (ret < 0 && ret != -ENOTSUP) {
+		rte_errno = ENOEXEC;
+		return -1;
+	}
+
 	rte_eal_mcfg_complete();
 
 	return fctret;
-- 
2.7.4

[PATCH v5 3/7] service cores: coremask parsing

[Harry van Haaren]

Add logic for parsing a coremask from EAL, which allows
the application to be unaware of the cores being taken from
its coremask.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

---

v4:
- Add --help print output (Jerin)
- Fixed coremask parsing to ensure master core is ROLE_RTE (Jerin)
- Improve coremask parsing error handling, master lcore and service coremask
  overlap is failed, informing user that the core cannot be used for the
  purpose A as it is already in use for B (A or B being service/master)

v2:
- Remove printf() (Jerin)
- Remove commented code (Jerin)
- simplified core tracking, no requirement on
  #include rte_service in EAL parsing anymore.

wip: fix master lcore handling in EAL
---
 lib/librte_eal/common/eal_common_options.c | 91 +++++++++++++++++++++++++++++-
 1 file changed, 90 insertions(+), 1 deletion(-)

diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index 075b0ea..00265d6 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -61,6 +61,7 @@ const char
 eal_short_options[] =
 	"b:" /* pci-blacklist */
 	"c:" /* coremask */
+	"s:" /* service coremask */
 	"d:" /* driver */
 	"h"  /* help */
 	"l:" /* corelist */
@@ -267,6 +268,77 @@ static int xdigit2val(unsigned char c)
 }
 
 static int
+eal_parse_service_coremask(const char *coremask)
+{
+	struct rte_config *cfg = rte_eal_get_configuration();
+	int i, j, idx = 0;
+	unsigned int count = 0;
+	char c;
+	int val;
+
+	if (coremask == NULL)
+		return -1;
+	/* Remove all blank characters ahead and after .
+	 * Remove 0x/0X if exists.
+	 */
+	while (isblank(*coremask))
+		coremask++;
+	if (coremask[0] == '0' && ((coremask[1] == 'x')
+		|| (coremask[1] == 'X')))
+		coremask += 2;
+	i = strlen(coremask);
+	while ((i > 0) && isblank(coremask[i - 1]))
+		i--;
+
+	if (i == 0)
+		return -1;
+
+	for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) {
+		c = coremask[i];
+		if (isxdigit(c) == 0) {
+			/* invalid characters */
+			return -1;
+		}
+		val = xdigit2val(c);
+		for (j = 0; j < BITS_PER_HEX && idx < RTE_MAX_LCORE;
+				j++, idx++) {
+			if ((1 << j) & val) {
+				/* handle master lcore already parsed */
+				uint32_t lcore = idx;
+				if (master_lcore_parsed &&
+						cfg->master_lcore == lcore) {
+					RTE_LOG(ERR, EAL,
+						"Error: lcore %u is master lcore, cannot use as service core\n",
+						idx);
+					return -1;
+				}
+
+				if (!lcore_config[idx].detected) {
+					RTE_LOG(ERR, EAL,
+						"lcore %u unavailable\n", idx);
+					return -1;
+				}
+				lcore_config[idx].core_role = ROLE_SERVICE;
+				count++;
+			}
+		}
+	}
+
+	for (; i >= 0; i--)
+		if (coremask[i] != '0')
+			return -1;
+
+	for (; idx < RTE_MAX_LCORE; idx++)
+		lcore_config[idx].core_index = -1;
+
+	if (count == 0)
+		return -1;
+
+	cfg->service_lcore_count = count;
+	return 0;
+}
+
+static int
 eal_parse_coremask(const char *coremask)
 {
 	struct rte_config *cfg = rte_eal_get_configuration();
@@ -409,6 +481,13 @@ eal_parse_master_lcore(const char *arg)
 	if (cfg->master_lcore >= RTE_MAX_LCORE)
 		return -1;
 	master_lcore_parsed = 1;
+
+	/* ensure master core is not used as service core */
+	if (lcore_config[cfg->master_lcore].core_role == ROLE_SERVICE) {
+		RTE_LOG(ERR, EAL, "Error: Master lcore is used as a service core.\n");
+		return -1;
+	}
+
 	return 0;
 }
 
@@ -826,6 +905,13 @@ eal_parse_common_option(int opt, const char *optarg,
 		}
 		core_parsed = 1;
 		break;
+	/* service coremask */
+	case 's':
+		if (eal_parse_service_coremask(optarg) < 0) {
+			RTE_LOG(ERR, EAL, "invalid service coremask\n");
+			return -1;
+		}
+		break;
 	/* size of memory */
 	case 'm':
 		conf->memory = atoi(optarg);
@@ -978,8 +1064,10 @@ eal_adjust_config(struct internal_config *internal_cfg)
 		internal_config.process_type = eal_proc_type_detect();
 
 	/* default master lcore is the first one */
-	if (!master_lcore_parsed)
+	if (!master_lcore_parsed) {
 		cfg->master_lcore = rte_get_next_lcore(-1, 0, 0);
+		lcore_config[cfg->master_lcore].core_role = ROLE_RTE;
+	}
 
 	/* if no memory amounts were requested, this will result in 0 and
 	 * will be overridden later, right after eal_hugepage_info_init() */
@@ -1045,6 +1133,7 @@ eal_common_usage(void)
 	       "                      ',' is used for single number separator.\n"
 	       "                      '( )' can be omitted for single element group,\n"
 	       "                      '@' can be omitted if cpus and lcores have the same value\n"
+	       "  -s SERVICE COREMASK Hexadecimal bitmask of cores to be used as service cores\n"
 	       "  --"OPT_MASTER_LCORE" ID   Core ID that is used as master\n"
 	       "  -n CHANNELS         Number of memory channels\n"
 	       "  -m MB               Memory to allocate (see also --"OPT_SOCKET_MEM")\n"
-- 
2.7.4

[PATCH v5 4/7] service cores: add unit tests

[Harry van Haaren]

Add a bunch of unit tests, to ensure that the service
core functions are operating as expected.

As part of these tests a dummy service is registered which
allows identifying if a service callback has been invoked
by using the CPU tick counter. This allows identifying if
functions to start and stop service lcores are actually having
effect.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

---

v5:
- Improve unit test for dump() to enable and disable stats (Jerin)
- Add new unit test for MT unsafe service callback (Harry)
- Added Ack from ML

v4:
- Remove static slcore_id, make service-core choice dynamic (Jerin)
- Use lcore_get_next() to acquire service cores (Jerin)
- Improve unit test to ensure app can remote_launc() on a core that
  previously served as a service-core (verifies ROLE_RTE is set) (Jerin)

v2 changes;
- Rename variable to slcore_id (Jerin)
- Rename function to unregister_all() (Jerin)
- Fix typos (Jerin)
- Add unit test for get_by_name()
- Add unit tests (all suggestions by Jerin)
-- get_name()
-- Verify probe_capability API
-- Verify MT_SAFE capability (see code for details)
-- Verify rte_service_dump() API
---
 test/test/Makefile             |   2 +
 test/test/test_service_cores.c | 599 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 601 insertions(+)
 create mode 100644 test/test/test_service_cores.c

diff --git a/test/test/Makefile b/test/test/Makefile
index e797c20..42d9a49 100644
--- a/test/test/Makefile
+++ b/test/test/Makefile
@@ -151,6 +151,8 @@ SRCS-y += test_interrupts.c
 SRCS-y += test_version.c
 SRCS-y += test_func_reentrancy.c
 
+SRCS-y += test_service_cores.c
+
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline.c
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_num.c
 SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += test_cmdline_etheraddr.c
diff --git a/test/test/test_service_cores.c b/test/test/test_service_cores.c
new file mode 100644
index 0000000..88fac8f
--- /dev/null
+++ b/test/test/test_service_cores.c
@@ -0,0 +1,599 @@
+/*-
+ *   BSD LICENSE
+ *
+ *   Copyright(c) 2017 Intel Corporation. All rights reserved.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Intel Corporation nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <rte_common.h>
+#include <rte_hexdump.h>
+#include <rte_mbuf.h>
+#include <rte_malloc.h>
+#include <rte_memcpy.h>
+#include <rte_cycles.h>
+
+#include <rte_service.h>
+#include <rte_service_component.h>
+
+#include "test.h"
+
+/* used as the service core ID */
+static uint32_t slcore_id;
+/* used as timestamp to detect if a service core is running */
+static uint64_t service_tick;
+/* used as a flag to check if a function was run */
+static uint32_t service_remote_launch_flag;
+
+#define SERVICE_DELAY 1
+
+#define DUMMY_SERVICE_NAME "dummy_service"
+#define MT_SAFE_SERVICE_NAME "mt_safe_service"
+
+static int
+testsuite_setup(void)
+{
+	slcore_id = rte_get_next_lcore(/* start core */ -1,
+				       /* skip master */ 1,
+				       /* wrap */ 0);
+
+	return TEST_SUCCESS;
+}
+
+static void
+testsuite_teardown(void)
+{
+	/* release service cores? */
+}
+
+static int32_t dummy_cb(void *args)
+{
+	RTE_SET_USED(args);
+	service_tick++;
+	rte_delay_ms(SERVICE_DELAY);
+	return 0;
+}
+
+static int32_t dummy_mt_unsafe_cb(void *args)
+{
+	/* before running test, the initialization has set pass_test to 1.
+	 * If the cmpset in service-cores is working correctly, the code here
+	 * should never fail to take the lock. If the lock *is* taken, fail the
+	 * test, because two threads are concurrently in a non-MT safe callback.
+	 */
+	uint32_t *test_params = args;
+	uint32_t *atomic_lock = &test_params[0];
+	uint32_t *pass_test = &test_params[1];
+	int lock_taken = rte_atomic32_cmpset(atomic_lock, 0, 1);
+	if (lock_taken) {
+		/* delay with the lock held */
+		rte_delay_ms(250);
+		rte_atomic32_clear((rte_atomic32_t *)atomic_lock);
+	} else {
+		/* 2nd thread will fail to take lock, so set pass flag */
+		*pass_test = 0;
+	}
+
+	return 0;
+}
+
+
+static int32_t dummy_mt_safe_cb(void *args)
+{
+	/* Atomic checks to ensure MT safe services allow > 1 thread to
+	 * concurrently run the callback. The concept is as follows;
+	 * 1) if lock is available, take the lock then delay
+	 * 2) if first lock is taken, and a thread arrives in the CB, we know
+	 *    that 2 threads are running the callback at the same time: MT safe
+	 */
+	uint32_t *test_params = args;
+	uint32_t *atomic_lock = &test_params[0];
+	uint32_t *pass_test = &test_params[1];
+	int lock_taken = rte_atomic32_cmpset(atomic_lock, 0, 1);
+	if (lock_taken) {
+		/* delay with the lock held */
+		rte_delay_ms(250);
+		rte_atomic32_clear((rte_atomic32_t *)atomic_lock);
+	} else {
+		/* 2nd thread will fail to take lock, so set pass flag */
+		*pass_test = 1;
+	}
+
+	return 0;
+}
+
+/* unregister all services */
+static int
+unregister_all(void)
+{
+	uint32_t i;
+	struct rte_service_spec *dead = (struct rte_service_spec *)0xdead;
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_unregister(0),
+			"Unregistered NULL pointer");
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_unregister(dead),
+			"Unregistered invalid pointer");
+
+	uint32_t c = rte_service_get_count();
+	for (i = 0; i < c; i++) {
+		struct rte_service_spec *s = rte_service_get_by_id(i);
+		TEST_ASSERT_EQUAL(0, rte_service_unregister(s),
+				"Error unregistering a valid service");
+	}
+
+	rte_service_lcore_reset_all();
+
+	return TEST_SUCCESS;
+}
+
+/* register a single dummy service */
+static int
+dummy_register(void)
+{
+	/* make sure there are no remains from previous tests */
+	unregister_all();
+
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid callback");
+	service.callback = dummy_cb;
+
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid name");
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Failed to register valid service");
+
+	return TEST_SUCCESS;
+}
+
+/* verify get_by_name() service lookup */
+static int
+service_get_by_name(void)
+{
+	unregister_all();
+
+	/* ensure with no services registered returns NULL */
+	TEST_ASSERT_EQUAL(0, rte_service_get_by_name(DUMMY_SERVICE_NAME),
+			"Service get by name should return NULL");
+
+	/* register service */
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid callback");
+	service.callback = dummy_cb;
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_register(&service),
+			"Invalid name");
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Failed to register valid service");
+
+	/* ensure with dummy services registered returns same ptr as ID */
+	struct rte_service_spec *s_by_id = rte_service_get_by_id(0);
+	TEST_ASSERT_EQUAL(s_by_id, rte_service_get_by_name(DUMMY_SERVICE_NAME),
+			"Service get_by_name should equal get_by_id()");
+
+	unregister_all();
+
+	/* ensure after unregister, get_by_name returns NULL */
+	TEST_ASSERT_EQUAL(0, rte_service_get_by_name(DUMMY_SERVICE_NAME),
+			"get by name should return NULL after unregister");
+
+	return TEST_SUCCESS;
+}
+
+/* verify probe of capabilities */
+static int
+service_probe_capability(void)
+{
+	unregister_all();
+
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	service.callback = dummy_cb;
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+	service.capabilities |= RTE_SERVICE_CAP_MT_SAFE;
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Register of MT SAFE service failed");
+
+	/* verify flag is enabled */
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+	int32_t mt = rte_service_probe_capability(s, RTE_SERVICE_CAP_MT_SAFE);
+	TEST_ASSERT_EQUAL(1, mt, "MT SAFE capability flag not set.");
+
+
+	unregister_all();
+
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	service.callback = dummy_cb;
+	snprintf(service.name, sizeof(service.name), DUMMY_SERVICE_NAME);
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Register of non-MT safe service failed");
+
+	/* verify flag is enabled */
+	s = rte_service_get_by_id(0);
+	mt = rte_service_probe_capability(s, RTE_SERVICE_CAP_MT_SAFE);
+	TEST_ASSERT_EQUAL(0, mt, "MT SAFE cap flag set on non MT SAFE service");
+
+	return unregister_all();
+}
+
+/* verify the service name */
+static int
+service_name(void)
+{
+	struct rte_service_spec *service = rte_service_get_by_id(0);
+
+	int equal = strcmp(service->name, DUMMY_SERVICE_NAME);
+	TEST_ASSERT_EQUAL(0, equal, "Error: Service name not correct");
+
+	return unregister_all();
+}
+
+/* verify service dump */
+static int
+service_dump(void)
+{
+	struct rte_service_spec *service = rte_service_get_by_id(0);
+	rte_service_set_stats_enable(service, 1);
+	rte_service_dump(stdout, service);
+	rte_service_set_stats_enable(service, 0);
+	rte_service_dump(stdout, service);
+	return unregister_all();
+}
+
+/* start and stop a service */
+static int
+service_start_stop(void)
+{
+	struct rte_service_spec *service = rte_service_get_by_id(0);
+
+	/* is_running() returns if service is running and slcore is mapped */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Service core add did not return zero");
+	int ret = rte_service_enable_on_lcore(service, slcore_id);
+	TEST_ASSERT_EQUAL(0, ret,
+			"Enabling service core, expected 0 got %d", ret);
+
+	TEST_ASSERT_EQUAL(0, rte_service_is_running(service),
+			"Error: Service should be stopped");
+
+	TEST_ASSERT_EQUAL(0, rte_service_stop(service),
+			"Error: Service stopped returned non-zero");
+
+	TEST_ASSERT_EQUAL(0, rte_service_is_running(service),
+			"Error: Service is running - should be stopped");
+
+	TEST_ASSERT_EQUAL(0, rte_service_start(service),
+			"Error: Service start returned non-zero");
+
+	TEST_ASSERT_EQUAL(1, rte_service_is_running(service),
+			"Error: Service is not running");
+
+	return unregister_all();
+}
+
+
+static int
+service_remote_launch_func(void *arg)
+{
+	RTE_SET_USED(arg);
+	service_remote_launch_flag = 1;
+	return 0;
+}
+
+/* enable and disable a lcore for a service */
+static int
+service_lcore_en_dis_able(void)
+{
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+
+	/* expected failure cases */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_enable_on_lcore(s, 100000),
+			"Enable on invalid core did not fail");
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_disable_on_lcore(s, 100000),
+			"Disable on invalid core did not fail");
+
+	/* add service core to allow enabling */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Add service core failed when not in use before");
+
+	/* valid enable */
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, slcore_id),
+			"Enabling valid service and core failed");
+	TEST_ASSERT_EQUAL(1, rte_service_get_enabled_on_lcore(s, slcore_id),
+			"Enabled core returned not-enabled");
+
+	/* valid disable */
+	TEST_ASSERT_EQUAL(0, rte_service_disable_on_lcore(s, slcore_id),
+			"Disabling valid service and lcore failed");
+	TEST_ASSERT_EQUAL(0, rte_service_get_enabled_on_lcore(s, slcore_id),
+			"Disabled core returned enabled");
+
+	/* call remote_launch to verify that app can launch ex-service lcore */
+	service_remote_launch_flag = 0;
+	int ret = rte_eal_remote_launch(service_remote_launch_func, NULL,
+					slcore_id);
+	TEST_ASSERT_EQUAL(0, ret, "Ex-service core remote launch failed.");
+	rte_eal_mp_wait_lcore();
+	TEST_ASSERT_EQUAL(1, service_remote_launch_flag,
+			"Ex-service core function call had no effect.");
+
+	return unregister_all();
+}
+
+static int
+service_lcore_running_check(void)
+{
+	uint64_t tick = service_tick;
+	rte_delay_ms(SERVICE_DELAY * 10);
+	/* if (tick != service_tick) we know the lcore as polled the service */
+	return tick != service_tick;
+}
+
+static int
+service_lcore_add_del(void)
+{
+	/* check initial count */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_count(),
+			"Service lcore count has value before adding a lcore");
+
+	/* check service lcore add */
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Add service core failed when not in use before");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_add(slcore_id),
+			"Add service core failed to refuse in-use lcore");
+
+	/* check count */
+	TEST_ASSERT_EQUAL(1, rte_service_lcore_count(),
+			"Service core count not equal to one");
+
+	/* retrieve core list, checking lcore ids */
+	const uint32_t size = 4;
+	uint32_t service_core_ids[size];
+	int32_t n = rte_service_lcore_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(1, n, "Service core list return should equal 1");
+	TEST_ASSERT_EQUAL(slcore_id, service_core_ids[0],
+				"Service core list lcore must equal slcore_id");
+
+	/* recheck count, add more cores, and check count */
+	TEST_ASSERT_EQUAL(1, rte_service_lcore_count(),
+			"Service core count not equal to one");
+	uint32_t slcore_1 = rte_get_next_lcore(/* start core */ -1,
+					       /* skip master */ 1,
+					       /* wrap */ 0);
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_1),
+			"Service core add did not return zero");
+	uint32_t slcore_2 = rte_get_next_lcore(/* start core */ slcore_1,
+					       /* skip master */ 1,
+					       /* wrap */ 0);
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_2),
+			"Service core add did not return zero");
+
+	uint32_t count = rte_service_lcore_count();
+	const uint32_t cores_at_this_point = 3;
+	TEST_ASSERT_EQUAL(cores_at_this_point, count,
+			"Service core count %d, expected %d", count,
+			cores_at_this_point);
+
+	/* check longer service core list */
+	n = rte_service_lcore_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(3, n, "Service core list return should equal 3");
+	TEST_ASSERT_EQUAL(slcore_id, service_core_ids[0],
+				"Service core list[0] lcore must equal 1");
+	TEST_ASSERT_EQUAL(slcore_1, service_core_ids[1],
+				"Service core list[1] lcore must equal 2");
+	TEST_ASSERT_EQUAL(slcore_2, service_core_ids[2],
+				"Service core list[2] lcore must equal 3");
+
+	/* recheck count, remove lcores, check remaining lcore_id is correct */
+	TEST_ASSERT_EQUAL(3, rte_service_lcore_count(),
+			"Service core count not equal to three");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_del(slcore_1),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_del(slcore_2),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(1, rte_service_lcore_count(),
+			"Service core count not equal to one");
+	n = rte_service_lcore_list(service_core_ids, size);
+	TEST_ASSERT_EQUAL(1, n, "Service core list return should equal one");
+	TEST_ASSERT_EQUAL(slcore_id, service_core_ids[0],
+				"Service core list[0] lcore must equal %d",
+				slcore_id);
+
+	return unregister_all();
+}
+
+static int
+service_threaded_test(int mt_safe)
+{
+	unregister_all();
+
+	/* add next 2 cores */
+	uint32_t slcore_1 = rte_get_next_lcore(/* start core */ -1,
+					       /* skip master */ 1,
+					       /* wrap */ 0);
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_1),
+			"mt safe lcore add fail");
+	uint32_t slcore_2 = rte_get_next_lcore(/* start core */ slcore_1,
+					       /* skip master */ 1,
+					       /* wrap */ 0);
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_2),
+			"mt safe lcore add fail");
+
+	/* Use atomic locks to verify that two threads are in the same function
+	 * at the same time. These are passed to the unit tests through the
+	 * callback userdata parameter
+	 */
+	uint32_t test_params[2];
+	memset(test_params, 0, sizeof(uint32_t) * 2);
+
+	/* register MT safe service. */
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	service.callback_userdata = test_params;
+	snprintf(service.name, sizeof(service.name), MT_SAFE_SERVICE_NAME);
+
+	if (mt_safe) {
+		service.callback = dummy_mt_safe_cb;
+		service.capabilities |= RTE_SERVICE_CAP_MT_SAFE;
+	} else {
+		/* initialize to pass, see callback comment for details */
+		test_params[1] = 1;
+		service.callback = dummy_mt_unsafe_cb;
+	}
+
+	TEST_ASSERT_EQUAL(0, rte_service_register(&service),
+			"Register of MT SAFE service failed");
+
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+	TEST_ASSERT_EQUAL(0, rte_service_start(s),
+			"Starting valid service failed");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, slcore_1),
+			"Failed to enable lcore 1 on mt safe service");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, slcore_2),
+			"Failed to enable lcore 2 on mt safe service");
+	rte_service_lcore_start(slcore_1);
+	rte_service_lcore_start(slcore_2);
+
+	/* wait for the worker threads to run */
+	rte_delay_ms(500);
+	rte_service_lcore_stop(slcore_1);
+	rte_service_lcore_stop(slcore_2);
+
+	TEST_ASSERT_EQUAL(1, test_params[1],
+			"MT Safe service not run by two cores concurrently");
+
+	TEST_ASSERT_EQUAL(0, rte_service_stop(s),
+			"Failed to stop MT Safe service");
+
+	unregister_all();
+
+	/* return the value of the callback pass_test variable to caller */
+	return test_params[1];
+}
+
+/* tests an MT SAFE service with two cores. The callback function ensures that
+ * two threads access the callback concurrently.
+ */
+static int
+service_mt_safe_poll(void)
+{
+	int mt_safe = 1;
+	TEST_ASSERT_EQUAL(1, service_threaded_test(mt_safe),
+			"Error: MT Safe service not run by two cores concurrently");
+	return TEST_SUCCESS;
+}
+
+/* tests a NON mt safe service with two cores, the callback is serialized
+ * using the atomic cmpset.
+ */
+static int
+service_mt_unsafe_poll(void)
+{
+	int mt_safe = 0;
+	TEST_ASSERT_EQUAL(1, service_threaded_test(mt_safe),
+			"Error: NON MT Safe service run by two cores concurrently");
+	return TEST_SUCCESS;
+}
+
+/* start and stop a service core - ensuring it goes back to sleep */
+static int
+service_lcore_start_stop(void)
+{
+	/* start service core and service, create mapping so tick() runs */
+	struct rte_service_spec *s = rte_service_get_by_id(0);
+	TEST_ASSERT_EQUAL(0, rte_service_start(s),
+			"Starting valid service failed");
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_enable_on_lcore(s, slcore_id),
+			"Enabling valid service on non-service core must fail");
+
+	/* core start */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_lcore_start(slcore_id),
+			"Service core start without add should return EINVAL");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_add(slcore_id),
+			"Service core add did not return zero");
+	TEST_ASSERT_EQUAL(0, rte_service_enable_on_lcore(s, slcore_id),
+			"Enabling valid service on valid core failed");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_start(slcore_id),
+			"Service core start after add failed");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_start(slcore_id),
+			"Service core expected as running but was stopped");
+
+	/* ensures core really is running the service function */
+	TEST_ASSERT_EQUAL(1, service_lcore_running_check(),
+			"Service core expected to poll service but it didn't");
+
+	/* core stop */
+	TEST_ASSERT_EQUAL(-EINVAL, rte_service_lcore_stop(100000),
+			"Invalid Service core stop should return -EINVAL");
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_stop(slcore_id),
+			"Service core stop expected to return 0");
+	TEST_ASSERT_EQUAL(-EALREADY, rte_service_lcore_stop(slcore_id),
+			"Already stopped service core should return -EALREADY");
+
+	/* ensure service is not longer running */
+	TEST_ASSERT_EQUAL(0, service_lcore_running_check(),
+			"Service core expected to poll service but it didn't");
+
+	TEST_ASSERT_EQUAL(0, rte_service_lcore_del(slcore_id),
+			"Service core del did not return zero");
+
+	return unregister_all();
+}
+
+static struct unit_test_suite service_tests  = {
+	.suite_name = "service core test suite",
+	.setup = testsuite_setup,
+	.teardown = testsuite_teardown,
+	.unit_test_cases = {
+		TEST_CASE_ST(dummy_register, NULL, unregister_all),
+		TEST_CASE_ST(dummy_register, NULL, service_name),
+		TEST_CASE_ST(dummy_register, NULL, service_get_by_name),
+		TEST_CASE_ST(dummy_register, NULL, service_dump),
+		TEST_CASE_ST(dummy_register, NULL, service_probe_capability),
+		TEST_CASE_ST(dummy_register, NULL, service_start_stop),
+		TEST_CASE_ST(dummy_register, NULL, service_lcore_add_del),
+		TEST_CASE_ST(dummy_register, NULL, service_lcore_start_stop),
+		TEST_CASE_ST(dummy_register, NULL, service_lcore_en_dis_able),
+		TEST_CASE_ST(dummy_register, NULL, service_mt_unsafe_poll),
+		TEST_CASE_ST(dummy_register, NULL, service_mt_safe_poll),
+		TEST_CASES_END() /**< NULL terminate unit test array */
+	}
+};
+
+static int
+test_service_common(void)
+{
+	return unit_test_suite_runner(&service_tests);
+}
+
+REGISTER_TEST_COMMAND(service_autotest, test_service_common);
-- 
2.7.4

[PATCH v5 5/7] event/sw: enable SW PMD with service capability

[Harry van Haaren]

This commit shows how easy it is to enable a specific
DPDK component with a service callback, in order to get
CPU cycles for it.

The beauty of this method is that the service is unaware
of how much CPU time it is getting - the application can
decide how to split and slice cores and map them to the
registered services.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

---

v4:
- Include Acked by

v2:
- Remove #include <rte_cycles.h> (Jerin)
- Remove development prints (Jerin)
- Track service name in PMD
- Print warning if service does not have an lcore mapped (Jerin)
---
 drivers/event/sw/sw_evdev.c | 32 ++++++++++++++++++++++++++++++++
 drivers/event/sw/sw_evdev.h |  3 +++
 2 files changed, 35 insertions(+)

diff --git a/drivers/event/sw/sw_evdev.c b/drivers/event/sw/sw_evdev.c
index 5b33bfd..9c534b7 100644
--- a/drivers/event/sw/sw_evdev.c
+++ b/drivers/event/sw/sw_evdev.c
@@ -39,6 +39,7 @@
 #include <rte_ring.h>
 #include <rte_errno.h>
 #include <rte_event_ring.h>
+#include <rte_service_component.h>
 
 #include "sw_evdev.h"
 #include "iq_ring.h"
@@ -614,6 +615,13 @@ sw_start(struct rte_eventdev *dev)
 {
 	unsigned int i, j;
 	struct sw_evdev *sw = sw_pmd_priv(dev);
+
+	/* check a service core is mapped to this service */
+	struct rte_service_spec *s = rte_service_get_by_name(sw->service_name);
+	if (!rte_service_is_running(s))
+		SW_LOG_ERR("Warning: No Service core enabled on service %s\n",
+				s->name);
+
 	/* check all ports are set up */
 	for (i = 0; i < sw->port_count; i++)
 		if (sw->ports[i].rx_worker_ring == NULL) {
@@ -716,6 +724,14 @@ set_credit_quanta(const char *key __rte_unused, const char *value, void *opaque)
 	return 0;
 }
 
+
+static int32_t sw_sched_service_func(void *args)
+{
+	struct rte_eventdev *dev = args;
+	sw_event_schedule(dev);
+	return 0;
+}
+
 static int
 sw_probe(struct rte_vdev_device *vdev)
 {
@@ -829,6 +845,22 @@ sw_probe(struct rte_vdev_device *vdev)
 	sw->credit_update_quanta = credit_quanta;
 	sw->sched_quanta = sched_quanta;
 
+	/* register service with EAL */
+	struct rte_service_spec service;
+	memset(&service, 0, sizeof(struct rte_service_spec));
+	snprintf(service.name, sizeof(service.name), "%s_service", name);
+	snprintf(sw->service_name, sizeof(sw->service_name), "%s_service",
+			name);
+	service.socket_id = socket_id;
+	service.callback = sw_sched_service_func;
+	service.callback_userdata = (void *)dev;
+
+	int32_t ret = rte_service_register(&service);
+	if (ret) {
+		SW_LOG_ERR("service register() failed");
+		return -ENOEXEC;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/event/sw/sw_evdev.h b/drivers/event/sw/sw_evdev.h
index 6ef03ce..71de3c1 100644
--- a/drivers/event/sw/sw_evdev.h
+++ b/drivers/event/sw/sw_evdev.h
@@ -59,6 +59,7 @@
 
 #define EVENTDEV_NAME_SW_PMD event_sw
 #define SW_PMD_NAME RTE_STR(event_sw)
+#define SW_PMD_NAME_MAX 64
 
 #define SW_SCHED_TYPE_DIRECT (RTE_SCHED_TYPE_PARALLEL + 1)
 
@@ -276,6 +277,8 @@ struct sw_evdev {
 	/* store num stats and offset of the stats for each queue */
 	uint16_t xstats_count_per_qid[RTE_EVENT_MAX_QUEUES_PER_DEV];
 	uint16_t xstats_offset_for_qid[RTE_EVENT_MAX_QUEUES_PER_DEV];
+
+	char service_name[SW_PMD_NAME_MAX];
 };
 
 static inline struct sw_evdev *
-- 
2.7.4

[PATCH v5 6/7] doc: add service cores to doc and release notes

[Harry van Haaren]

Add a section describing the fundamental concepts behind service
cores. Where service cores originate from, and how to enable services.
The release notes for 17.08 are updated, with an introductory paragraph
on the service cores concept. Finally the Eventdev SW PMD documentation
is amended to reflect that it can be run as a service.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

---

v4:
- Add ack from ML

I would like to enable the service-cores in the eventdev_pipeline
sample app, to showcase the power of the service-core abstraction.
There is some remaining work TODO, in order to genericise the sample
app for both HW and SW PMDs, and during that rework the service-cores
can be added too.

The sample app will make a good showcase for docs, and make it much
easier to understand.
---
 doc/guides/eventdevs/sw.rst             |  4 +-
 doc/guides/prog_guide/index.rst         |  1 +
 doc/guides/prog_guide/service_cores.rst | 81 +++++++++++++++++++++++++++++++++
 doc/guides/rel_notes/release_17_08.rst  |  8 ++++
 4 files changed, 93 insertions(+), 1 deletion(-)
 create mode 100644 doc/guides/prog_guide/service_cores.rst

diff --git a/doc/guides/eventdevs/sw.rst b/doc/guides/eventdevs/sw.rst
index fb63c84..a3e6624 100644
--- a/doc/guides/eventdevs/sw.rst
+++ b/doc/guides/eventdevs/sw.rst
@@ -32,7 +32,9 @@ Software Eventdev Poll Mode Driver
 
 The software eventdev is an implementation of the eventdev API, that provides a
 wide range of the eventdev features. The eventdev relies on a CPU core to
-perform event scheduling.
+perform event scheduling. This PMD can use the service core library to run the
+scheduling function, allowing an application to utilize the power of service
+cores to multiplex other work on the same core if required.
 
 
 Features
diff --git a/doc/guides/prog_guide/index.rst b/doc/guides/prog_guide/index.rst
index 7578395..5548aba 100644
--- a/doc/guides/prog_guide/index.rst
+++ b/doc/guides/prog_guide/index.rst
@@ -38,6 +38,7 @@ Programmer's Guide
     intro
     overview
     env_abstraction_layer
+    service_cores
     ring_lib
     mempool_lib
     mbuf_lib
diff --git a/doc/guides/prog_guide/service_cores.rst b/doc/guides/prog_guide/service_cores.rst
new file mode 100644
index 0000000..3a029ba
--- /dev/null
+++ b/doc/guides/prog_guide/service_cores.rst
@@ -0,0 +1,81 @@
+..  BSD LICENSE
+    Copyright(c) 2017 Intel Corporation. All rights reserved.
+    All rights reserved.
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions
+    are met:
+
+    * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+    * Neither the name of Intel Corporation nor the names of its
+    contributors may be used to endorse or promote products derived
+    from this software without specific prior written permission.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+Service Cores
+=============
+
+DPDK has a concept known as service cores, which enables a dynamic way of
+performing work on DPDK lcores. Service core support is built into the EAL, and
+an API is provided to optionally allow applications to control how the service
+cores are used at runtime.
+
+The service cores concept is built up out of services (components of DPDK that
+require CPU cycles to operate) and service cores (DPDK lcores, tasked with
+running services). The power of the service core concept is that the mapping
+between service cores and services can be configured to abstract away the
+difference between platforms and environments.
+
+For example, the Eventdev has hardware and software PMDs. Of these the software
+PMD requires an lcore to perform the scheduling operations, while the hardware
+PMD does not. With service cores, the application would not directly notice
+that the scheduling is done in software.
+
+For detailed information about the service core API, please refer to the docs.
+
+Service Core Initialization
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are two methods to having service cores in a DPDK application, either by
+using the service coremask, or by dynamically adding cores using the API.
+The simpler of the two is to pass the `-s` coremask argument to EAL, which will
+take any cores available in the main DPDK coremask, an if the bits are also set
+in the service coremask the cores become service-cores instead of DPDK
+application lcores.
+
+Enabling Services on Cores
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Each registered service can be individually mapped to a service core, or set of
+service cores. Enabling a service on a particular core means that the lcore in
+question will run the service. Disabling that core on the service stops the
+lcore in question from running the service.
+
+Using this method, it is possible to assign specific workloads to each
+service core, and map N workloads to M number of service cores. Each service
+lcore loops over the services that are enabled for that core, and invokes the
+function to run the service.
+
+Service Core Statistics
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The service core library is capable of collecting runtime statistics like number
+of calls to a specific service, and number of cycles used by the service. The
+cycle count collection is dynamically configurable, allowing any application to
+profile the services running on the system at any time.
diff --git a/doc/guides/rel_notes/release_17_08.rst b/doc/guides/rel_notes/release_17_08.rst
index 6273098..ab68f6f 100644
--- a/doc/guides/rel_notes/release_17_08.rst
+++ b/doc/guides/rel_notes/release_17_08.rst
@@ -59,6 +59,14 @@ New Features
     RAW pattern items with QUEUE actions. There are four
     type of filter support for this feature on igb.
 
+* **Added Service Core functionality.**
+
+  The service core functionality added to EAL allows DPDK to run services such
+  as SW PMDs on lcores without the application manually running them. The
+  service core infrastructure allows flexibility of running multiple services
+  on the same service lcore, and provides the application with powerful APIs to
+  configure the mapping from service lcores to services.
+
 * **Added Generic Flow API support to enic.**
 
   Flow API support for outer Ethernet, VLAN, IPv4, IPv6, UDP, TCP, SCTP, VxLAN
-- 
2.7.4

[PATCH v5 7/7] maintainers: claim service cores

[Harry van Haaren]

Sign-up to be the maintainer of public header files and
implementation of the service-cores infrastructure.

Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

---

v5:
- Mark with EXPERIMENTAL
- Update filename to rte_service_component.h
- Added Ack from ML

v4:
- fix rte_service.c filename
- Include service_cores.rst file
---
 MAINTAINERS | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/MAINTAINERS b/MAINTAINERS
index 804ac04..b7df49b 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -134,6 +134,13 @@ F: test/test/test_mp_secondary.c
 F: examples/multi_process/
 F: doc/guides/sample_app_ug/multi_process.rst
 
+Service Cores - EXPERIMENTAL
+M: Harry van Haaren <harry.van.haaren@intel.com>
+F: doc/guides/prog_guide/service_cores.rst
+F: lib/librte_eal/common/include/rte_service.h
+F: lib/librte_eal/common/include/rte_service_component.h
+F: lib/librte_eal/common/rte_service.c
+
 ARM v7
 M: Jan Viktorin <viktorin@rehivetech.com>
 M: Jianbo Liu <jianbo.liu@linaro.org>
-- 
2.7.4

Re: [PATCH v5 1/7] service cores: header and implementation

[Jerin Jacob]

-----Original Message-----
> Date: Tue, 11 Jul 2017 15:19:27 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: thomas@monjalon.net, jerin.jacob@caviumnetworks.com,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH v5 1/7] service cores: header and implementation
> X-Mailer: git-send-email 2.7.4
> 
> Add header files, update .map files with new service
> functions, and add the service header to the doxygen
> for building.
> 
> This service header API allows DPDK to use services as
> a concept of something that requires CPU cycles. An example
> is a PMD that runs in software to schedule events, where a
> hardware version exists that does not require a CPU.
> 
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

Looks good to me.

Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

> 
> ---
> 
> v5:
> - Improved service_set_stats_enable() to operate per service (Jerin)
> - Fixed un-documented doxygen parameter (Jerin)
> - Renamed cores_state to lcore_states (Jerin)
> - Optimized atomic operations and flags (Jerin)
> - Removed info about RFCs etc from commit message (Jerin)
> - Add lcore_count check to default setup function and return early (Jerin)
> - Add memory barriers to lcore_add() and lcore_del() (Jerin)
> - Rename start function to rte_service_start_with_defaults() (Jerin)
> - Rename header to rte_service_component.h (Jerin/Thomas)
> 
> v4:
> - Fixed (unsigned) checkpatch error
> - Fixed misleading-indentation/if { } brackets (checkpatch/Jerin)
> - Fixed set function argument to be "enable" instead of "enabled" (Jerin)
> - Improve doxygen comment for size of array in rte_service_core_list (Jerin)
> - Fixed typos (Jerin)
> - Optimized atomic clear after running service (Jerin)
> - Added smp_rmb() at end of loop to re-load runstate / mapping (Jerin)
> - Fix issue with lcore role not being adhered to (Jerin)
> - Add experimental warnings for all service core functions (Thomas)
> - Moved service core functions into EXPERIMENTAL section of .map (Thomas)
> - Improve documentation of rte_service_lcore_reset_all() (Harry)
> 
> v3:
> - None.
> 
> v2:
> Thanks Jerin for review - below a list your suggested changes;
> - Doxygen rename to "service cores" for consistency
> - use lcore instead of core for function names
> - Fix about 10 typos / seplling msitakse ;)
> - Dix doxygen /** comments for functions
> - Doxygen @param[out] improvements
> - int8_t for socket_id to ordinary int
> - Rename MACROS for readability
> - Align structs to cache lines
> - Allocate fastpath-used data from hugepages
> - Added/fixed memory barriers for multi-core scheduling
> - Add const to variables, and hoist above loop
> - Optimize cmpset atomic if MT_SAFE or only one core mapped
> - Statistics collection only when requested
> - Add error check for array pointer
> - Remove panic() calls from library
> - Fix TODO notes from previous patchset
> 
> There are also some other changes;
> - Checkpatch issues fixed
> - .map file updates
> - Add rte_service_get_by_name() function

Re: [PATCH v5 0/7] service cores: cover letter

[Jerin Jacob]

-----Original Message-----
> Date: Tue, 11 Jul 2017 15:19:26 +0100
> From: Harry van Haaren <harry.van.haaren@intel.com>
> To: dev@dpdk.org
> CC: thomas@monjalon.net, jerin.jacob@caviumnetworks.com,
>  keith.wiles@intel.com, bruce.richardson@intel.com, Harry van Haaren
>  <harry.van.haaren@intel.com>
> Subject: [PATCH v5 0/7] service cores: cover letter
> X-Mailer: git-send-email 2.7.4
> 
> This patchset introduces service cores to DPDK. A service core
> is an lcore that performs functions to abstract away details of
> differences in environment of the application.
> 
> An example is using the eventdev API, where either a software or hardware
> PMD performs scheduling. In the case of the software PMD an lcore is
> required to perform scheduling, which means application logic would have
> to be aware of the PMD running under the API. To abstract away the
> differences in HW / SW PMDs, service cores can run the SW PMD service
> without application logic specifying the exact cores to use. Note that
> eventdev is only one API that benefits; timers, interrupts handling,
> statistics and monitoring, and a range of other infrastructure that
> requires a slice of CPU time may all benefit from service cores.
> 
> The application is not obliged to manually use the service cores API,
> however if an application wishes to use the service cores API for fine
> grained control over how the services are run, this is possible. Deciding
> between a performance threading-profile and scaled-down profile can be
> achieved by advanced usage of service cores and setting the lcore mappings.
> 
> Patch 5/7 shows how a PMD can register a service to run a function. This
> is then available (along with any other registered services) to be run by
> the service cores. Patches 6/7 and 7/7 add documentation, and claim
> maintainership.
> 
> Regards, -Harry

Nice feature. This series Looks good to me.
Series Acked-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>

IMO, We can add this series in RC2 if there are no objections.

Two points:

1) There is a check-git log warning. It could be fixed it on apply.

Wrong headline uppercase:
	service cores: EAL init changes


2) Based on the mail[1], It is agreed that we need to refactor eal_parse_service_coremask() and
eal_parse__coremask() code in future to avoid code duplication in coremask parsing
logic.

[1]
http://dpdk.org/ml/archives/dev/2017-July/070610.html

> 
> v5:
> Jerin:
> - Fix documentation warnings
> - Rename variables to better names
> - Enable statistics per-service
> - Improve atomic operation flag checks
> - Reworked function to rte_service_start_with_defaults()
> - Added memory barriers to lcore_add() and lcore_del()
> - Simplified EAL code, reduced duplication and makes it more maintainable
> 
> Jerin/Thomas:
> - Rename component header to rte_service_component.h
> 
> v4:
> - Range of fixes as suggested by Jerin
> - Improved unit tests, ensuring ex-service cores become available to app
> - Added functions to EXPERIMENTAL tag in .map files (Thomas)
> - Added @warning experimental notes to Doxygen API documentation (Thomas)
> - Various smaller fixes / cleanups
> - See commit notes for details
> 
> v3:
> - Added docs
> - Added release notes
> - Updated maintainers file
> - Compile checks with devtools/test-build.sh
> - Validated patches apply to latest dpdk/master
> - Based on discussion, rte_service_iterate() is *not* included,
>   but could be adding at a later date if use-cases require it.
> - Future work includes enabling the eventdev_pipeline sample app, but there
>   is still some churn there to enable both HW/SW PMDs seamlessly. Once sample
>   app is enabled a service core walk-through with that sample app can be added
>   to the docs, to provide a tutorial on service-core usage.
> 
> 
> Harry van Haaren (7):
>   service cores: header and implementation
>   service cores: EAL init changes
>   service cores: coremask parsing
>   service cores: add unit tests
>   event/sw: enable SW PMD with service capability
>   doc: add service cores to doc and release notes
>   maintainers: claim service cores
> 
>  MAINTAINERS                                        |   7 +
>  doc/api/doxy-api-index.md                          |   1 +
>  doc/guides/eventdevs/sw.rst                        |   4 +-
>  doc/guides/prog_guide/index.rst                    |   1 +
>  doc/guides/prog_guide/service_cores.rst            |  81 +++
>  doc/guides/rel_notes/release_17_08.rst             |   8 +
>  drivers/event/sw/sw_evdev.c                        |  32 +
>  drivers/event/sw/sw_evdev.h                        |   3 +
>  lib/librte_eal/bsdapp/eal/Makefile                 |   1 +
>  lib/librte_eal/bsdapp/eal/eal.c                    |  18 +
>  lib/librte_eal/bsdapp/eal/rte_eal_version.map      |  23 +
>  lib/librte_eal/common/Makefile                     |   1 +
>  lib/librte_eal/common/eal_common_lcore.c           |   1 +
>  lib/librte_eal/common/eal_common_options.c         |  91 ++-
>  lib/librte_eal/common/include/rte_eal.h            |   4 +
>  lib/librte_eal/common/include/rte_lcore.h          |   3 +-
>  lib/librte_eal/common/include/rte_service.h        | 387 +++++++++++
>  .../common/include/rte_service_component.h         | 144 +++++
>  lib/librte_eal/common/rte_service.c                | 704 +++++++++++++++++++++
>  lib/librte_eal/linuxapp/eal/Makefile               |   1 +
>  lib/librte_eal/linuxapp/eal/eal.c                  |  18 +
>  lib/librte_eal/linuxapp/eal/eal_thread.c           |   9 +-
>  lib/librte_eal/linuxapp/eal/rte_eal_version.map    |  23 +
>  test/test/Makefile                                 |   2 +
>  test/test/test_service_cores.c                     | 599 ++++++++++++++++++
>  25 files changed, 2162 insertions(+), 4 deletions(-)
>  create mode 100644 doc/guides/prog_guide/service_cores.rst
>  create mode 100644 lib/librte_eal/common/include/rte_service.h
>  create mode 100644 lib/librte_eal/common/include/rte_service_component.h
>  create mode 100644 lib/librte_eal/common/rte_service.c
>  create mode 100644 test/test/test_service_cores.c
> 
> -- 
> 2.7.4
> 

Re: [PATCH v5 0/7] service cores: cover letter

[Thomas Monjalon]

Hi Harry,

I like the service concept!

11/07/2017 16:19, Harry van Haaren:
> Harry van Haaren (7):
>   service cores: header and implementation
>   service cores: EAL init changes
>   service cores: coremask parsing
>   service cores: add unit tests
>   event/sw: enable SW PMD with service capability
>   doc: add service cores to doc and release notes
>   maintainers: claim service cores

Titles are shorten to "service:" instead of "service cores".
No need of specific patches for doc and maintainers.
I've squashed them appropriately.

The unit test file is also added in the maintainers section.

Applied, thanks

Next step: I think you should convert service coremask to a corelist
option. We must stop using coremasks because they are limited.

Re: [PATCH v5 0/7] service cores: cover letter

[Van Haaren, Harry]

> From: Thomas Monjalon [mailto:thomas@monjalon.net]
> Sent: Sunday, July 16, 2017 8:26 PM
> To: Van Haaren, Harry <harry.van.haaren@intel.com>
> Cc: dev@dpdk.org; jerin.jacob@caviumnetworks.com; Wiles, Keith <keith.wiles@intel.com>;
> Richardson, Bruce <bruce.richardson@intel.com>
> Subject: Re: [dpdk-dev] [PATCH v5 0/7] service cores: cover letter
> 
> Hi Harry,
> 
> I like the service concept!

Cool!
 
> 11/07/2017 16:19, Harry van Haaren:
> > Harry van Haaren (7):
> >   service cores: header and implementation
> >   service cores: EAL init changes
> >   service cores: coremask parsing
> >   service cores: add unit tests
> >   event/sw: enable SW PMD with service capability
> >   doc: add service cores to doc and release notes
> >   maintainers: claim service cores
> 
> Titles are shorten to "service:" instead of "service cores".
> No need of specific patches for doc and maintainers.
> I've squashed them appropriately.
> 
> The unit test file is also added in the maintainers section.
> 
> Applied, thanks
> 
> Next step: I think you should convert service coremask to a corelist
> option. We must stop using coremasks because they are limited.

Thanks for fixups, and agreed on corelist - I'll work on getting a patch up.

[PATCH] service: add corelist to EAL arguments

[Harry van Haaren]

This commit allows the -S (captial 's') to be used to indicate
a corelist for Services. This is a "nice to have" patch, and does
not modify any of the service core functionality.

Suggested-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>
Suggested-by: Thomas Monjalon <thomas@monjalon.net>
Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
---
 lib/librte_eal/common/eal_common_options.c | 74 ++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)

diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
index 00265d6..696a627 100644
--- a/lib/librte_eal/common/eal_common_options.c
+++ b/lib/librte_eal/common/eal_common_options.c
@@ -65,6 +65,7 @@ eal_short_options[] =
 	"d:" /* driver */
 	"h"  /* help */
 	"l:" /* corelist */
+	"S:" /* service corelist */
 	"m:" /* memory size */
 	"n:" /* memory channels */
 	"r:" /* memory ranks */
@@ -402,6 +403,72 @@ eal_parse_coremask(const char *coremask)
 }
 
 static int
+eal_parse_service_corelist(const char *corelist)
+{
+	struct rte_config *cfg = rte_eal_get_configuration();
+	int i, idx = 0;
+	unsigned count = 0;
+	char *end = NULL;
+	int min, max;
+
+	if (corelist == NULL)
+		return -1;
+
+	/* Remove all blank characters ahead and after */
+	while (isblank(*corelist))
+		corelist++;
+	i = strlen(corelist);
+	while ((i > 0) && isblank(corelist[i - 1]))
+		i--;
+
+	/* Get list of cores */
+	min = RTE_MAX_LCORE;
+	do {
+		while (isblank(*corelist))
+			corelist++;
+		if (*corelist == '\0')
+			return -1;
+		errno = 0;
+		idx = strtoul(corelist, &end, 10);
+		if (errno || end == NULL)
+			return -1;
+		while (isblank(*end))
+			end++;
+		if (*end == '-') {
+			min = idx;
+		} else if ((*end == ',') || (*end == '\0')) {
+			max = idx;
+			if (min == RTE_MAX_LCORE)
+				min = idx;
+			for (idx = min; idx <= max; idx++) {
+				if (cfg->lcore_role[idx] != ROLE_SERVICE) {
+					/* handle master lcore already parsed */
+					uint32_t lcore = idx;
+					if (cfg->master_lcore == lcore &&
+							master_lcore_parsed) {
+						RTE_LOG(ERR, EAL,
+							"Error: lcore %u is master lcore, cannot use as service core\n",
+							idx);
+						return -1;
+					}
+					lcore_config[idx].core_role =
+							ROLE_SERVICE;
+					count++;
+				}
+			}
+			min = RTE_MAX_LCORE;
+		} else
+			return -1;
+		corelist = end + 1;
+	} while (*end != '\0');
+
+	if (count == 0)
+		return -1;
+
+	return 0;
+}
+
+static int
 eal_parse_corelist(const char *corelist)
 {
 	struct rte_config *cfg = rte_eal_get_configuration();
@@ -912,6 +979,13 @@ eal_parse_common_option(int opt, const char *optarg,
 			return -1;
 		}
 		break;
+	/* service corelist */
+	case 'S':
+		if (eal_parse_service_corelist(optarg) < 0) {
+			RTE_LOG(ERR, EAL, "invalid service core list\n");
+			return -1;
+		}
+		break;
 	/* size of memory */
 	case 'm':
 		conf->memory = atoi(optarg);
-- 
2.7.4

Re: [PATCH] service: add corelist to EAL arguments

[Ananyev, Konstantin]

Hi Harry,

> -----Original Message-----
> From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Harry van Haaren
> Sent: Monday, July 17, 2017 4:22 PM
> To: dev@dpdk.org
> Cc: thomas@monjalon.net; jerin.jacob@caviumnetworks.com; Van Haaren, Harry <harry.van.haaren@intel.com>
> Subject: [dpdk-dev] [PATCH] service: add corelist to EAL arguments
> 
> This commit allows the -S (captial 's') to be used to indicate
> a corelist for Services. This is a "nice to have" patch, and does
> not modify any of the service core functionality.
> 
> Suggested-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>
> Suggested-by: Thomas Monjalon <thomas@monjalon.net>
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> ---
>  lib/librte_eal/common/eal_common_options.c | 74 ++++++++++++++++++++++++++++++
>  1 file changed, 74 insertions(+)
> 
> diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
> index 00265d6..696a627 100644
> --- a/lib/librte_eal/common/eal_common_options.c
> +++ b/lib/librte_eal/common/eal_common_options.c
> @@ -65,6 +65,7 @@ eal_short_options[] =
>  	"d:" /* driver */
>  	"h"  /* help */
>  	"l:" /* corelist */
> +	"S:" /* service corelist */
>  	"m:" /* memory size */
>  	"n:" /* memory channels */
>  	"r:" /* memory ranks */
> @@ -402,6 +403,72 @@ eal_parse_coremask(const char *coremask)
>  }
> 

Do we need a new parsing function here?
Can't we reuse at least part of '-l' parsing code?
Konstantin

>  static int
> +eal_parse_service_corelist(const char *corelist)
> +{
> +	struct rte_config *cfg = rte_eal_get_configuration();
> +	int i, idx = 0;
> +	unsigned count = 0;
> +	char *end = NULL;
> +	int min, max;
> +
> +	if (corelist == NULL)
> +		return -1;
> +
> +	/* Remove all blank characters ahead and after */
> +	while (isblank(*corelist))
> +		corelist++;
> +	i = strlen(corelist);
> +	while ((i > 0) && isblank(corelist[i - 1]))
> +		i--;
> +
> +	/* Get list of cores */
> +	min = RTE_MAX_LCORE;
> +	do {
> +		while (isblank(*corelist))
> +			corelist++;
> +		if (*corelist == '\0')
> +			return -1;
> +		errno = 0;
> +		idx = strtoul(corelist, &end, 10);
> +		if (errno || end == NULL)
> +			return -1;
> +		while (isblank(*end))
> +			end++;
> +		if (*end == '-') {
> +			min = idx;
> +		} else if ((*end == ',') || (*end == '\0')) {
> +			max = idx;
> +			if (min == RTE_MAX_LCORE)
> +				min = idx;
> +			for (idx = min; idx <= max; idx++) {
> +				if (cfg->lcore_role[idx] != ROLE_SERVICE) {
> +					/* handle master lcore already parsed */
> +					uint32_t lcore = idx;
> +					if (cfg->master_lcore == lcore &&
> +							master_lcore_parsed) {
> +						RTE_LOG(ERR, EAL,
> +							"Error: lcore %u is master lcore, cannot use as service core\n",
> +							idx);
> +						return -1;
> +					}
> +					lcore_config[idx].core_role =
> +							ROLE_SERVICE;
> +					count++;
> +				}
> +			}
> +			min = RTE_MAX_LCORE;
> +		} else
> +			return -1;
> +		corelist = end + 1;
> +	} while (*end != '\0');
> +
> +	if (count == 0)
> +		return -1;
> +
> +	return 0;
> +}
> +
> +static int
>  eal_parse_corelist(const char *corelist)
>  {
>  	struct rte_config *cfg = rte_eal_get_configuration();
> @@ -912,6 +979,13 @@ eal_parse_common_option(int opt, const char *optarg,
>  			return -1;
>  		}
>  		break;
> +	/* service corelist */
> +	case 'S':
> +		if (eal_parse_service_corelist(optarg) < 0) {
> +			RTE_LOG(ERR, EAL, "invalid service core list\n");
> +			return -1;
> +		}
> +		break;
>  	/* size of memory */
>  	case 'm':
>  		conf->memory = atoi(optarg);
> --
> 2.7.4

Re: [PATCH] service: add corelist to EAL arguments

[Van Haaren, Harry]

> From: Ananyev, Konstantin
> Sent: Monday, July 17, 2017 4:54 PM
> To: Van Haaren, Harry <harry.van.haaren@intel.com>; dev@dpdk.org
> Cc: thomas@monjalon.net; jerin.jacob@caviumnetworks.com; Van Haaren, Harry
> <harry.van.haaren@intel.com>
> Subject: RE: [dpdk-dev] [PATCH] service: add corelist to EAL arguments
> 
> Hi Harry,
> 
> > -----Original Message-----
> > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Harry van Haaren
> > Sent: Monday, July 17, 2017 4:22 PM
> > To: dev@dpdk.org
> > Cc: thomas@monjalon.net; jerin.jacob@caviumnetworks.com; Van Haaren, Harry
> <harry.van.haaren@intel.com>
> > Subject: [dpdk-dev] [PATCH] service: add corelist to EAL arguments
> >
> > This commit allows the -S (captial 's') to be used to indicate
> > a corelist for Services. This is a "nice to have" patch, and does
> > not modify any of the service core functionality.
> >
> > Suggested-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>
> > Suggested-by: Thomas Monjalon <thomas@monjalon.net>
> > Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> > ---
> >  lib/librte_eal/common/eal_common_options.c | 74 ++++++++++++++++++++++++++++++
> >  1 file changed, 74 insertions(+)
> >
> > diff --git a/lib/librte_eal/common/eal_common_options.c
> b/lib/librte_eal/common/eal_common_options.c
> > index 00265d6..696a627 100644
> > --- a/lib/librte_eal/common/eal_common_options.c
> > +++ b/lib/librte_eal/common/eal_common_options.c
> > @@ -65,6 +65,7 @@ eal_short_options[] =
> >  	"d:" /* driver */
> >  	"h"  /* help */
> >  	"l:" /* corelist */
> > +	"S:" /* service corelist */
> >  	"m:" /* memory size */
> >  	"n:" /* memory channels */
> >  	"r:" /* memory ranks */
> > @@ -402,6 +403,72 @@ eal_parse_coremask(const char *coremask)
> >  }
> >
> 
> Do we need a new parsing function here?
> Can't we reuse at least part of '-l' parsing code?
> Konstantin


Yep we should - in this instance (post-RC2..) I don't want to rock-the-boat and change any existing EAL functionality.

During review of eventdev-sample-app and service-cores, Jerin had noted that we are duplicating various functions for parsing strings, and really we should cleanup into a single function to call from all of them. That should be scheduled as post 17.08 rework.

Re: [PATCH] service: add corelist to EAL arguments

[Ananyev, Konstantin]

> -----Original Message-----
> From: Van Haaren, Harry
> Sent: Monday, July 17, 2017 4:58 PM
> To: Ananyev, Konstantin <konstantin.ananyev@intel.com>; dev@dpdk.org
> Cc: thomas@monjalon.net; jerin.jacob@caviumnetworks.com
> Subject: RE: [dpdk-dev] [PATCH] service: add corelist to EAL arguments
> 
> > From: Ananyev, Konstantin
> > Sent: Monday, July 17, 2017 4:54 PM
> > To: Van Haaren, Harry <harry.van.haaren@intel.com>; dev@dpdk.org
> > Cc: thomas@monjalon.net; jerin.jacob@caviumnetworks.com; Van Haaren, Harry
> > <harry.van.haaren@intel.com>
> > Subject: RE: [dpdk-dev] [PATCH] service: add corelist to EAL arguments
> >
> > Hi Harry,
> >
> > > -----Original Message-----
> > > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Harry van Haaren
> > > Sent: Monday, July 17, 2017 4:22 PM
> > > To: dev@dpdk.org
> > > Cc: thomas@monjalon.net; jerin.jacob@caviumnetworks.com; Van Haaren, Harry
> > <harry.van.haaren@intel.com>
> > > Subject: [dpdk-dev] [PATCH] service: add corelist to EAL arguments
> > >
> > > This commit allows the -S (captial 's') to be used to indicate
> > > a corelist for Services. This is a "nice to have" patch, and does
> > > not modify any of the service core functionality.
> > >
> > > Suggested-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>
> > > Suggested-by: Thomas Monjalon <thomas@monjalon.net>
> > > Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> > > ---
> > >  lib/librte_eal/common/eal_common_options.c | 74 ++++++++++++++++++++++++++++++
> > >  1 file changed, 74 insertions(+)
> > >
> > > diff --git a/lib/librte_eal/common/eal_common_options.c
> > b/lib/librte_eal/common/eal_common_options.c
> > > index 00265d6..696a627 100644
> > > --- a/lib/librte_eal/common/eal_common_options.c
> > > +++ b/lib/librte_eal/common/eal_common_options.c
> > > @@ -65,6 +65,7 @@ eal_short_options[] =
> > >  	"d:" /* driver */
> > >  	"h"  /* help */
> > >  	"l:" /* corelist */
> > > +	"S:" /* service corelist */
> > >  	"m:" /* memory size */
> > >  	"n:" /* memory channels */
> > >  	"r:" /* memory ranks */
> > > @@ -402,6 +403,72 @@ eal_parse_coremask(const char *coremask)
> > >  }
> > >
> >
> > Do we need a new parsing function here?
> > Can't we reuse at least part of '-l' parsing code?
> > Konstantin
> 
> 
> Yep we should - in this instance (post-RC2..) I don't want to rock-the-boat and change any existing EAL functionality.
> 
> During review of eventdev-sample-app and service-cores, Jerin had noted that we are duplicating various functions for parsing strings, and
> really we should cleanup into a single function to call from all of them. That should be scheduled as post 17.08 rework.

Sound ok to me then.
Konstantin

Re: [PATCH] service: add corelist to EAL arguments

[Van Haaren, Harry]

> From: Ananyev, Konstantin
> Sent: Monday, July 17, 2017 5:11 PM
> To: Van Haaren, Harry <harry.van.haaren@intel.com>; dev@dpdk.org
> Cc: thomas@monjalon.net; jerin.jacob@caviumnetworks.com
> Subject: RE: [dpdk-dev] [PATCH] service: add corelist to EAL arguments
> 
> 
> 
> > -----Original Message-----
> > From: Van Haaren, Harry
> > Sent: Monday, July 17, 2017 4:58 PM
> > To: Ananyev, Konstantin <konstantin.ananyev@intel.com>; dev@dpdk.org
> > Cc: thomas@monjalon.net; jerin.jacob@caviumnetworks.com
> > Subject: RE: [dpdk-dev] [PATCH] service: add corelist to EAL arguments
> >
> > > From: Ananyev, Konstantin
> > > Sent: Monday, July 17, 2017 4:54 PM
> > > To: Van Haaren, Harry <harry.van.haaren@intel.com>; dev@dpdk.org
> > > Cc: thomas@monjalon.net; jerin.jacob@caviumnetworks.com; Van Haaren, Harry
> > > <harry.van.haaren@intel.com>
> > > Subject: RE: [dpdk-dev] [PATCH] service: add corelist to EAL arguments
> > >
> > > Hi Harry,
> > >
> > > > -----Original Message-----
> > > > From: dev [mailto:dev-bounces@dpdk.org] On Behalf Of Harry van Haaren
> > > > Sent: Monday, July 17, 2017 4:22 PM
> > > > To: dev@dpdk.org
> > > > Cc: thomas@monjalon.net; jerin.jacob@caviumnetworks.com; Van Haaren, Harry
> > > <harry.van.haaren@intel.com>
> > > > Subject: [dpdk-dev] [PATCH] service: add corelist to EAL arguments
> > > >
> > > > This commit allows the -S (captial 's') to be used to indicate
> > > > a corelist for Services. This is a "nice to have" patch, and does
> > > > not modify any of the service core functionality.
> > > >
> > > > Suggested-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>
> > > > Suggested-by: Thomas Monjalon <thomas@monjalon.net>
> > > > Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>
> > > > ---
> > > >  lib/librte_eal/common/eal_common_options.c | 74 ++++++++++++++++++++++++++++++
> > > >  1 file changed, 74 insertions(+)
> > > >
> > > > diff --git a/lib/librte_eal/common/eal_common_options.c
> > > b/lib/librte_eal/common/eal_common_options.c
> > > > index 00265d6..696a627 100644
> > > > --- a/lib/librte_eal/common/eal_common_options.c
> > > > +++ b/lib/librte_eal/common/eal_common_options.c
> > > > @@ -65,6 +65,7 @@ eal_short_options[] =
> > > >  	"d:" /* driver */
> > > >  	"h"  /* help */
> > > >  	"l:" /* corelist */
> > > > +	"S:" /* service corelist */
> > > >  	"m:" /* memory size */
> > > >  	"n:" /* memory channels */
> > > >  	"r:" /* memory ranks */
> > > > @@ -402,6 +403,72 @@ eal_parse_coremask(const char *coremask)
> > > >  }
> > > >
> > >
> > > Do we need a new parsing function here?
> > > Can't we reuse at least part of '-l' parsing code?
> > > Konstantin
> >
> >
> > Yep we should - in this instance (post-RC2..) I don't want to rock-the-boat and change
> any existing EAL functionality.
> >
> > During review of eventdev-sample-app and service-cores, Jerin had noted that we are
> duplicating various functions for parsing strings, and
> > really we should cleanup into a single function to call from all of them. That should be
> scheduled as post 17.08 rework.
> 
> Sound ok to me then.

Cool. I should note; good point, absolutely agree, and thanks for review :)

Re: [PATCH] service: add corelist to EAL arguments

[Thomas Monjalon]

17/07/2017 18:21, Harry van Haaren:
> This commit allows the -S (captial 's') to be used to indicate
> a corelist for Services. This is a "nice to have" patch, and does
> not modify any of the service core functionality.
> 
> Suggested-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>
> Suggested-by: Thomas Monjalon <thomas@monjalon.net>
> Signed-off-by: Harry van Haaren <harry.van.haaren@intel.com>

Applied, thanks