From: Ian Betts <ian.betts@intel.com>
To: dev@dpdk.org
Cc: Ian Betts <ian.betts@intel.com>
Subject: [PATCH v7 2/4] examples: add lthread subsystem for performance-thread
Date: Thu, 3 Dec 2015 16:21:21 +0000 [thread overview]
Message-ID: <1449159683-7092-3-git-send-email-ian.betts@intel.com> (raw)
In-Reply-To: <1449159683-7092-1-git-send-email-ian.betts@intel.com>
This commit adds the lightweight thread subsystem used by the
performance-thread sample applications.
Signed-off-by: Ian Betts <ian.betts@intel.com>
---
.../performance-thread/common/arch/x86/atomic.h | 59 ++
examples/performance-thread/common/arch/x86/ctx.c | 93 +++
examples/performance-thread/common/arch/x86/ctx.h | 57 ++
examples/performance-thread/common/common.mk | 42 ++
examples/performance-thread/common/lthread.c | 530 +++++++++++++
examples/performance-thread/common/lthread.h | 99 +++
examples/performance-thread/common/lthread_api.h | 829 +++++++++++++++++++++
examples/performance-thread/common/lthread_cond.c | 241 ++++++
examples/performance-thread/common/lthread_cond.h | 77 ++
examples/performance-thread/common/lthread_diag.c | 321 ++++++++
examples/performance-thread/common/lthread_diag.h | 129 ++++
.../performance-thread/common/lthread_diag_api.h | 319 ++++++++
examples/performance-thread/common/lthread_int.h | 212 ++++++
examples/performance-thread/common/lthread_mutex.c | 256 +++++++
examples/performance-thread/common/lthread_mutex.h | 52 ++
.../performance-thread/common/lthread_objcache.h | 160 ++++
examples/performance-thread/common/lthread_pool.h | 333 +++++++++
examples/performance-thread/common/lthread_queue.h | 303 ++++++++
examples/performance-thread/common/lthread_sched.c | 600 +++++++++++++++
examples/performance-thread/common/lthread_sched.h | 152 ++++
examples/performance-thread/common/lthread_timer.h | 78 ++
examples/performance-thread/common/lthread_tls.c | 254 +++++++
examples/performance-thread/common/lthread_tls.h | 57 ++
23 files changed, 5253 insertions(+)
create mode 100644 examples/performance-thread/common/arch/x86/atomic.h
create mode 100644 examples/performance-thread/common/arch/x86/ctx.c
create mode 100644 examples/performance-thread/common/arch/x86/ctx.h
create mode 100644 examples/performance-thread/common/common.mk
create mode 100644 examples/performance-thread/common/lthread.c
create mode 100644 examples/performance-thread/common/lthread.h
create mode 100644 examples/performance-thread/common/lthread_api.h
create mode 100644 examples/performance-thread/common/lthread_cond.c
create mode 100644 examples/performance-thread/common/lthread_cond.h
create mode 100644 examples/performance-thread/common/lthread_diag.c
create mode 100644 examples/performance-thread/common/lthread_diag.h
create mode 100644 examples/performance-thread/common/lthread_diag_api.h
create mode 100644 examples/performance-thread/common/lthread_int.h
create mode 100644 examples/performance-thread/common/lthread_mutex.c
create mode 100644 examples/performance-thread/common/lthread_mutex.h
create mode 100644 examples/performance-thread/common/lthread_objcache.h
create mode 100644 examples/performance-thread/common/lthread_pool.h
create mode 100644 examples/performance-thread/common/lthread_queue.h
create mode 100644 examples/performance-thread/common/lthread_sched.c
create mode 100644 examples/performance-thread/common/lthread_sched.h
create mode 100644 examples/performance-thread/common/lthread_timer.h
create mode 100644 examples/performance-thread/common/lthread_tls.c
create mode 100644 examples/performance-thread/common/lthread_tls.h
diff --git a/examples/performance-thread/common/arch/x86/atomic.h b/examples/performance-thread/common/arch/x86/atomic.h
new file mode 100644
index 0000000..d77becf
--- /dev/null
+++ b/examples/performance-thread/common/arch/x86/atomic.h
@@ -0,0 +1,59 @@
+/*
+ *-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef ATOMIC_H_
+#define ATOMIC_H_
+
+#include <stdint.h>
+
+/*
+ * Atomically set a value and return the old value
+ */
+static inline uint64_t
+atomic64_xchg(uint64_t *ptr, uint64_t val) __attribute__ ((always_inline));
+static inline uint64_t
+atomic64_xchg(uint64_t *ptr, uint64_t val)
+{
+ asm volatile (
+ "lock;"
+ "xchgq %0,%1;"
+ : "=r" ((uint64_t) val)
+ : "m" (*(uint64_t *) ptr), "0" (val)
+ : "memory");
+
+ return val;
+}
+
+
+#endif /* ATOMIC_H_ */
diff --git a/examples/performance-thread/common/arch/x86/ctx.c b/examples/performance-thread/common/arch/x86/ctx.c
new file mode 100644
index 0000000..1e8e271
--- /dev/null
+++ b/examples/performance-thread/common/arch/x86/ctx.c
@@ -0,0 +1,93 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * https://github.com/halayli/lthread which carries the following license.
+ *
+ * Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+
+
+#if defined(__x86_64__)
+__asm__ (
+".text\n"
+".p2align 4,,15\n"
+".globl ctx_switch\n"
+".globl _ctx_switch\n"
+"ctx_switch:\n"
+"_ctx_switch:\n"
+" movq %rsp, 0(%rsi) # save stack_pointer\n"
+" movq %rbp, 8(%rsi) # save frame_pointer\n"
+" movq (%rsp), %rax # save insn_pointer\n"
+" movq %rax, 16(%rsi)\n"
+" movq %rbx, 24(%rsi)\n # save rbx,r12-r15\n"
+" movq 24(%rdi), %rbx\n"
+" movq %r15, 56(%rsi)\n"
+" movq %r14, 48(%rsi)\n"
+" movq 48(%rdi), %r14\n"
+" movq 56(%rdi), %r15\n"
+" movq %r13, 40(%rsi)\n"
+" movq %r12, 32(%rsi)\n"
+" movq 32(%rdi), %r12\n"
+" movq 40(%rdi), %r13\n"
+" movq 0(%rdi), %rsp # restore stack_pointer\n"
+" movq 16(%rdi), %rax # restore insn_pointer\n"
+" movq 8(%rdi), %rbp # restore frame_pointer\n"
+" movq %rax, (%rsp)\n"
+" ret\n"
+ );
+#else
+#pragma GCC error "__x86_64__ is not defined"
+#endif
diff --git a/examples/performance-thread/common/arch/x86/ctx.h b/examples/performance-thread/common/arch/x86/ctx.h
new file mode 100644
index 0000000..0386050
--- /dev/null
+++ b/examples/performance-thread/common/arch/x86/ctx.h
@@ -0,0 +1,57 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#ifndef CTX_H
+#define CTX_H
+
+/*
+ * CPU context registers
+ */
+struct ctx {
+ void *rsp; /* 0 */
+ void *rbp; /* 8 */
+ void *rip; /* 16 */
+ void *rbx; /* 24 */
+ void *r12; /* 32 */
+ void *r13; /* 40 */
+ void *r14; /* 48 */
+ void *r15; /* 56 */
+};
+
+
+void
+ctx_switch(struct ctx *new_ctx, struct ctx *curr_ctx);
+
+
+#endif /* RTE_CTX_H_ */
diff --git a/examples/performance-thread/common/common.mk b/examples/performance-thread/common/common.mk
new file mode 100644
index 0000000..d3de5fc
--- /dev/null
+++ b/examples/performance-thread/common/common.mk
@@ -0,0 +1,42 @@
+#
+# BSD LICENSE
+#
+# Copyright(c) 2015 Intel Corporation. All rights reserved.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Intel Corporation nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# list the C files belonhing to the lthread subsystem, these are common to all lthread apps
+SRCS-y += ../common/lthread.c \
+ ../common/lthread_sched.c \
+ ../common/lthread_cond.c \
+ ../common/lthread_tls.c \
+ ../common/lthread_mutex.c \
+ ../common/lthread_diag.c \
+ ../common/arch/x86/ctx.c
+
+INCLUDES += -I$(RTE_SDK)/examples/performance-thread/common/ -I$(RTE_SDK)/examples/performance-thread/common/arch/x86/
diff --git a/examples/performance-thread/common/lthread.c b/examples/performance-thread/common/lthread.c
new file mode 100644
index 0000000..e8addb7
--- /dev/null
+++ b/examples/performance-thread/common/lthread.c
@@ -0,0 +1,530 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Some portions of this software is derived from the
+ * https://github.com/halayli/lthread which carrys the following license.
+ *
+ * Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#define RTE_MEM 1
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <limits.h>
+#include <inttypes.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <fcntl.h>
+#include <sys/time.h>
+#include <sys/mman.h>
+
+#include <rte_config.h>
+#include <rte_log.h>
+#include <ctx.h>
+
+#include "lthread_api.h"
+#include "lthread.h"
+#include "lthread_timer.h"
+#include "lthread_tls.h"
+#include "lthread_objcache.h"
+#include "lthread_diag.h"
+
+
+/*
+ * This function gets called after an lthread function has returned.
+ */
+void _lthread_exit_handler(struct lthread *lt)
+{
+
+ lt->state |= BIT(ST_LT_EXITED);
+
+ if (!(lt->state & BIT(ST_LT_DETACH))) {
+ /* thread is this not explicitly detached
+ * it must be joinable, so we call lthread_exit().
+ */
+ lthread_exit(NULL);
+ }
+
+ /* if we get here the thread is detached so we can reschedule it,
+ * allowing the scheduler to free it
+ */
+ _reschedule();
+}
+
+
+/*
+ * Free resources allocated to an lthread
+ */
+void _lthread_free(struct lthread *lt)
+{
+
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_FREE, lt, 0);
+
+ /* invoke any user TLS destructor functions */
+ _lthread_tls_destroy(lt);
+
+ /* free memory allocated for TLS defined using RTE_PER_LTHREAD macros */
+ if (sizeof(void *) < (uint64_t)RTE_PER_LTHREAD_SECTION_SIZE)
+ _lthread_objcache_free(lt->tls->root_sched->per_lthread_cache,
+ lt->per_lthread_data);
+
+ /* free pthread style TLS memory */
+ _lthread_objcache_free(lt->tls->root_sched->tls_cache, lt->tls);
+
+ /* free the stack */
+ _lthread_objcache_free(lt->stack_container->root_sched->stack_cache,
+ lt->stack_container);
+
+ /* now free the thread */
+ _lthread_objcache_free(lt->root_sched->lthread_cache, lt);
+
+}
+
+/*
+ * Allocate a stack and maintain a cache of stacks
+ */
+struct lthread_stack *_stack_alloc(void)
+{
+ struct lthread_stack *s;
+
+ s = _lthread_objcache_alloc((THIS_SCHED)->stack_cache);
+ LTHREAD_ASSERT(s != NULL);
+
+ s->root_sched = THIS_SCHED;
+ s->stack_size = LTHREAD_MAX_STACK_SIZE;
+ return s;
+}
+
+/*
+ * Execute a ctx by invoking the start function
+ * On return call an exit handler if the user has provided one
+ */
+static void _lthread_exec(void *arg)
+{
+ struct lthread *lt = (struct lthread *)arg;
+
+ /* invoke the contexts function */
+ lt->fun(lt->arg);
+ /* do exit handling */
+ if (lt->exit_handler != NULL)
+ lt->exit_handler(lt);
+}
+
+/*
+ * Initialize an lthread
+ * Set its function, args, and exit handler
+ */
+void
+_lthread_init(struct lthread *lt,
+ lthread_func_t fun, void *arg, lthread_exit_func exit_handler)
+{
+
+ /* set ctx func and args */
+ lt->fun = fun;
+ lt->arg = arg;
+ lt->exit_handler = exit_handler;
+
+ /* set initial state */
+ lt->birth = _sched_now();
+ lt->state = BIT(ST_LT_INIT);
+ lt->join = LT_JOIN_INITIAL;
+}
+
+/*
+ * set the lthread stack
+ */
+void _lthread_set_stack(struct lthread *lt, void *stack, size_t stack_size)
+{
+ char *stack_top = (char *)stack + stack_size;
+ void **s = (void **)stack_top;
+
+ /* set stack */
+ lt->stack = stack;
+ lt->stack_size = stack_size;
+
+ /* set initial context */
+ s[-3] = NULL;
+ s[-2] = (void *)lt;
+ lt->ctx.rsp = (void *)(stack_top - (4 * sizeof(void *)));
+ lt->ctx.rbp = (void *)(stack_top - (3 * sizeof(void *)));
+ lt->ctx.rip = (void *)_lthread_exec;
+}
+
+/*
+ * Create an lthread on the current scheduler
+ * If there is no current scheduler on this pthread then first create one
+ */
+int
+lthread_create(struct lthread **new_lt, int lcore_id,
+ lthread_func_t fun, void *arg)
+{
+ if ((new_lt == NULL) || (fun == NULL))
+ return POSIX_ERRNO(EINVAL);
+
+ if (lcore_id < 0)
+ lcore_id = rte_lcore_id();
+ else if (lcore_id > LTHREAD_MAX_LCORES)
+ return POSIX_ERRNO(EINVAL);
+
+ struct lthread *lt = NULL;
+
+ if (THIS_SCHED == NULL) {
+ THIS_SCHED = _lthread_sched_create(0);
+ if (THIS_SCHED == NULL) {
+ perror("Failed to create scheduler");
+ return POSIX_ERRNO(EAGAIN);
+ }
+ }
+
+ /* allocate a thread structure */
+ lt = _lthread_objcache_alloc((THIS_SCHED)->lthread_cache);
+ if (lt == NULL)
+ return POSIX_ERRNO(EAGAIN);
+
+ bzero(lt, sizeof(struct lthread));
+ lt->root_sched = THIS_SCHED;
+
+ /* set the function args and exit handlder */
+ _lthread_init(lt, fun, arg, _lthread_exit_handler);
+
+ /* put it in the ready queue */
+ *new_lt = lt;
+
+ if (lcore_id < 0)
+ lcore_id = rte_lcore_id();
+
+ DIAG_CREATE_EVENT(lt, LT_DIAG_LTHREAD_CREATE);
+
+ rte_wmb();
+ _ready_queue_insert(_lthread_sched_get(lcore_id), lt);
+ return 0;
+}
+
+/*
+ * Schedules lthread to sleep for `nsecs`
+ * setting the lthread state to LT_ST_SLEEPING.
+ * lthread state is cleared upon resumption or expiry.
+ */
+static inline void _lthread_sched_sleep(struct lthread *lt, uint64_t nsecs)
+{
+ uint64_t state = lt->state;
+ uint64_t clks = _ns_to_clks(nsecs);
+
+ if (clks) {
+ _timer_start(lt, clks);
+ lt->state = state | BIT(ST_LT_SLEEPING);
+ }
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_SLEEP, clks, 0);
+ _suspend();
+}
+
+
+
+/*
+ * Cancels any running timer.
+ * This can be called multiple times on the same lthread regardless if it was
+ * sleeping or not.
+ */
+int _lthread_desched_sleep(struct lthread *lt)
+{
+ uint64_t state = lt->state;
+
+ if (state & BIT(ST_LT_SLEEPING)) {
+ _timer_stop(lt);
+ state &= (CLEARBIT(ST_LT_SLEEPING) & CLEARBIT(ST_LT_EXPIRED));
+ lt->state = state | BIT(ST_LT_READY);
+ return 1;
+ }
+ return 0;
+}
+
+/*
+ * set user data pointer in an lthread
+ */
+void lthread_set_data(void *data)
+{
+ if (sizeof(void *) == RTE_PER_LTHREAD_SECTION_SIZE)
+ THIS_LTHREAD->per_lthread_data = data;
+}
+
+/*
+ * Retrieve user data pointer from an lthread
+ */
+void *lthread_get_data(void)
+{
+ return THIS_LTHREAD->per_lthread_data;
+}
+
+/*
+ * Return the current lthread handle
+ */
+struct lthread *lthread_current(void)
+{
+ struct lthread_sched *sched = THIS_SCHED;
+
+ if (sched)
+ return sched->current_lthread;
+ return NULL;
+}
+
+
+
+/*
+ * Tasklet to cancel a thread
+ */
+static void
+_cancel(void *arg)
+{
+ struct lthread *lt = (struct lthread *) arg;
+
+ lt->state |= BIT(ST_LT_CANCELLED);
+ lthread_detach();
+}
+
+
+/*
+ * Mark the specified as canceled
+ */
+int lthread_cancel(struct lthread *cancel_lt)
+{
+ struct lthread *lt;
+
+ if ((cancel_lt == NULL) || (cancel_lt == THIS_LTHREAD))
+ return POSIX_ERRNO(EINVAL);
+
+ DIAG_EVENT(cancel_lt, LT_DIAG_LTHREAD_CANCEL, cancel_lt, 0);
+
+ if (cancel_lt->sched != THIS_SCHED) {
+
+ /* spawn task-let to cancel the thread */
+ lthread_create(<,
+ cancel_lt->sched->lcore_id,
+ _cancel,
+ cancel_lt);
+ return 0;
+ }
+ cancel_lt->state |= BIT(ST_LT_CANCELLED);
+ return 0;
+}
+
+/*
+ * Suspend the current lthread for specified time
+ */
+void lthread_sleep(uint64_t nsecs)
+{
+ struct lthread *lt = THIS_LTHREAD;
+
+ _lthread_sched_sleep(lt, nsecs);
+
+}
+
+/*
+ * Suspend the current lthread for specified time
+ */
+void lthread_sleep_clks(uint64_t clks)
+{
+ struct lthread *lt = THIS_LTHREAD;
+ uint64_t state = lt->state;
+
+ if (clks) {
+ _timer_start(lt, clks);
+ lt->state = state | BIT(ST_LT_SLEEPING);
+ }
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_SLEEP, clks, 0);
+ _suspend();
+}
+
+/*
+ * Requeue the current thread to the back of the ready queue
+ */
+void lthread_yield(void)
+{
+ struct lthread *lt = THIS_LTHREAD;
+
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_YIELD, 0, 0);
+
+ _ready_queue_insert(THIS_SCHED, lt);
+ ctx_switch(&(THIS_SCHED)->ctx, <->ctx);
+}
+
+/*
+ * Exit the current lthread
+ * If a thread is joining pass the user pointer to it
+ */
+void lthread_exit(void *ptr)
+{
+ struct lthread *lt = THIS_LTHREAD;
+
+ /* if thread is detached (this is not valid) just exit */
+ if (lt->state & BIT(ST_LT_DETACH))
+ return;
+
+ /* There is a race between lthread_join() and lthread_exit()
+ * - if exit before join then we suspend and resume on join
+ * - if join before exit then we resume the joining thread
+ */
+ if ((lt->join == LT_JOIN_INITIAL)
+ && rte_atomic64_cmpset(<->join, LT_JOIN_INITIAL,
+ LT_JOIN_EXITING)) {
+
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_EXIT, 1, 0);
+ _suspend();
+ /* set the exit value */
+ if ((ptr != NULL) && (lt->lt_join->lt_exit_ptr != NULL))
+ *(lt->lt_join->lt_exit_ptr) = ptr;
+
+ /* let the joining thread know we have set the exit value */
+ lt->join = LT_JOIN_EXIT_VAL_SET;
+ } else {
+
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_EXIT, 0, 0);
+ /* set the exit value */
+ if ((ptr != NULL) && (lt->lt_join->lt_exit_ptr != NULL))
+ *(lt->lt_join->lt_exit_ptr) = ptr;
+ /* let the joining thread know we have set the exit value */
+ lt->join = LT_JOIN_EXIT_VAL_SET;
+ _ready_queue_insert(lt->lt_join->sched,
+ (struct lthread *)lt->lt_join);
+ }
+
+
+ /* wait until the joinging thread has collected the exit value */
+ while (lt->join != LT_JOIN_EXIT_VAL_READ)
+ _reschedule();
+
+ /* reset join state */
+ lt->join = LT_JOIN_INITIAL;
+
+ /* detach it so its resources can be released */
+ lt->state |= (BIT(ST_LT_DETACH) | BIT(ST_LT_EXITED));
+}
+
+/*
+ * Join an lthread
+ * Suspend until the joined thread returns
+ */
+int lthread_join(struct lthread *lt, void **ptr)
+{
+ if (lt == NULL)
+ return POSIX_ERRNO(EINVAL);
+
+ struct lthread *current = THIS_LTHREAD;
+ uint64_t lt_state = lt->state;
+
+ /* invalid to join a detached thread, or a thread that is joined */
+ if ((lt_state & BIT(ST_LT_DETACH)) || (lt->join == LT_JOIN_THREAD_SET))
+ return POSIX_ERRNO(EINVAL);
+ /* pointer to the joining thread and a poingter to return a value */
+ lt->lt_join = current;
+ current->lt_exit_ptr = ptr;
+ /* There is a race between lthread_join() and lthread_exit()
+ * - if join before exit we suspend and will resume when exit is called
+ * - if exit before join we resume the exiting thread
+ */
+ if ((lt->join == LT_JOIN_INITIAL)
+ && rte_atomic64_cmpset(<->join, LT_JOIN_INITIAL,
+ LT_JOIN_THREAD_SET)) {
+
+ DIAG_EVENT(current, LT_DIAG_LTHREAD_JOIN, lt, 1);
+ _suspend();
+ } else {
+ DIAG_EVENT(current, LT_DIAG_LTHREAD_JOIN, lt, 0);
+ _ready_queue_insert(lt->sched, lt);
+ }
+
+ /* wait for exiting thread to set return value */
+ while (lt->join != LT_JOIN_EXIT_VAL_SET)
+ _reschedule();
+
+ /* collect the return value */
+ if (ptr != NULL)
+ *ptr = *current->lt_exit_ptr;
+
+ /* let the exiting thread proceed to exit */
+ lt->join = LT_JOIN_EXIT_VAL_READ;
+ return 0;
+}
+
+
+/*
+ * Detach current lthread
+ * A detached thread cannot be joined
+ */
+void lthread_detach(void)
+{
+ struct lthread *lt = THIS_LTHREAD;
+
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_DETACH, 0, 0);
+
+ uint64_t state = lt->state;
+
+ lt->state = state | BIT(ST_LT_DETACH);
+}
+
+/*
+ * Set function name of an lthread
+ * this is a debug aid
+ */
+void lthread_set_funcname(const char *f)
+{
+ struct lthread *lt = THIS_LTHREAD;
+
+ strncpy(lt->funcname, f, sizeof(lt->funcname));
+ lt->funcname[sizeof(lt->funcname)-1] = 0;
+}
diff --git a/examples/performance-thread/common/lthread.h b/examples/performance-thread/common/lthread.h
new file mode 100644
index 0000000..8c77af8
--- /dev/null
+++ b/examples/performance-thread/common/lthread.h
@@ -0,0 +1,99 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Some portions of this software is derived from the
+ * https://github.com/halayli/lthread which carrys the following license.
+ *
+ * Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef LTHREAD_H_
+#define LTHREAD_H_
+
+#include <rte_per_lcore.h>
+
+#include "lthread_api.h"
+#include "lthread_diag.h"
+
+struct lthread;
+struct lthread_sched;
+
+/* function to be called when a context function returns */
+typedef void (*lthread_exit_func) (struct lthread *);
+
+void _lthread_exit_handler(struct lthread *lt);
+
+void lthread_set_funcname(const char *f);
+
+void _lthread_sched_busy_sleep(struct lthread *lt, uint64_t nsecs);
+
+int _lthread_desched_sleep(struct lthread *lt);
+
+void _lthread_free(struct lthread *lt);
+
+struct lthread_sched *_lthread_sched_get(int lcore_id);
+
+struct lthread_stack *_stack_alloc(void);
+
+struct
+lthread_sched *_lthread_sched_create(size_t stack_size);
+
+void
+_lthread_init(struct lthread *lt,
+ lthread_func_t fun, void *arg, lthread_exit_func exit_handler);
+
+void _lthread_set_stack(struct lthread *lt, void *stack, size_t stack_size);
+
+#endif /* LTHREAD_H_ */
diff --git a/examples/performance-thread/common/lthread_api.h b/examples/performance-thread/common/lthread_api.h
new file mode 100644
index 0000000..e5e7dcc
--- /dev/null
+++ b/examples/performance-thread/common/lthread_api.h
@@ -0,0 +1,829 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Some portions of this software may have been derived from the
+ * https://github.com/halayli/lthread which carrys the following license.
+ *
+ * Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/**
+ * @file
+ *
+ * This file contains the public API for the L-thread subsystem
+ *
+ * The L_thread subsystem provides a simple cooperative scheduler to
+ * enable arbitrary functions to run as cooperative threads within a
+ * single P-thread.
+ *
+ * The subsystem provides a P-thread like API that is intended to assist in
+ * reuse of legacy code written for POSIX p_threads.
+ *
+ * The L-thread subsystem relies on cooperative multitasking, as such
+ * an L-thread must possess frequent rescheduling points. Often these
+ * rescheduling points are provided transparently when the application
+ * invokes an L-thread API.
+ *
+ * In some applications it is possible that the program may enter a loop the
+ * exit condition for which depends on the action of another thread or a
+ * response from hardware. In such a case it is necessary to yield the thread
+ * periodically in the loop body, to allow other threads an opportunity to
+ * run. This can be done by inserting a call to lthread_yield() or
+ * lthread_sleep(n) in the body of the loop.
+ *
+ * If the application makes expensive / blocking system calls or does other
+ * work that would take an inordinate amount of time to complete, this will
+ * stall the cooperative scheduler resulting in very poor performance.
+ *
+ * In such cases an L-thread can be migrated temporarily to another scheduler
+ * running in a different P-thread on another core. When the expensive or
+ * blocking operation is completed it can be migrated back to the original
+ * scheduler. In this way other threads can continue to run on the original
+ * scheduler and will be completely unaffected by the blocking behaviour.
+ * To migrate an L-thread to another scheduler the API lthread_set_affinity()
+ * is provided.
+ *
+ * If L-threads that share data are running on the same core it is possible
+ * to design programs where mutual exclusion mechanisms to protect shared data
+ * can be avoided. This is due to the fact that the cooperative threads cannot
+ * preempt each other.
+ *
+ * There are two cases where mutual exclusion mechanisms are necessary.
+ *
+ * a) Where the L-threads sharing data are running on different cores.
+ * b) Where code must yield while updating data shared with another thread.
+ *
+ * The L-thread subsystem provides a set of mutex APIs to help with such
+ * scenarios, however excessive reliance on on these will impact performance
+ * and is best avoided if possible.
+ *
+ * L-threads can synchronise using a fast condition variable implementation
+ * that supports signal and broadcast. An L-thread running on any core can
+ * wait on a condition.
+ *
+ * L-threads can have L-thread local storage with an API modelled on either the
+ * P-thread get/set specific API or using PER_LTHREAD macros modelled on the
+ * RTE_PER_LCORE macros. Alternatively a simple user data pointer may be set
+ * and retrieved from a thread.
+ */
+#ifndef LTHREAD_H
+#define LTHREAD_H
+
+#include <stdint.h>
+#include <sys/socket.h>
+#include <fcntl.h>
+#include <netinet/in.h>
+
+#include <rte_cycles.h>
+
+
+struct lthread;
+struct lthread_cond;
+struct lthread_mutex;
+
+struct lthread_condattr;
+struct lthread_mutexattr;
+
+typedef void (*lthread_func_t) (void *);
+
+/*
+ * Define the size of stack for an lthread
+ * Then this is the size that will be allocated on lthread creation
+ * This is a fixed size and will not grow.
+ */
+#define LTHREAD_MAX_STACK_SIZE (1024*64)
+
+/**
+ * Define the maximum number of TLS keys that can be created
+ *
+ */
+#define LTHREAD_MAX_KEYS 1024
+
+/**
+ * Define the maximum number of attempts to destroy an lthread's
+ * TLS data on thread exit
+ */
+#define LTHREAD_DESTRUCTOR_ITERATIONS 4
+
+
+/**
+ * Define the maximum number of lcores that will support lthreads
+ */
+#define LTHREAD_MAX_LCORES RTE_MAX_LCORE
+
+/**
+ * How many lthread objects to pre-allocate as the system grows
+ * applies to lthreads + stacks, TLS, mutexs, cond vars.
+ *
+ * @see _lthread_alloc()
+ * @see _cond_alloc()
+ * @see _mutex_alloc()
+ *
+ */
+#define LTHREAD_PREALLOC 100
+
+/**
+ * Set the number of schedulers in the system.
+ *
+ * This function may optionally be called before starting schedulers.
+ *
+ * If the number of schedulers is not set, or set to 0 then each scheduler
+ * will begin scheduling lthreads immediately it is started.
+
+ * If the number of schedulers is set to greater than 0, then each scheduler
+ * will wait until all schedulers have started before beginning to schedule
+ * lthreads.
+ *
+ * If an application wishes to have threads migrate between cores using
+ * lthread_set_affinity(), or join threads running on other cores using
+ * lthread_join(), then it is prudent to set the number of schedulers to ensure
+ * that all schedulers are initialised beforehand.
+ *
+ * @param num
+ * the number of schedulers in the system
+ * @return
+ * the number of schedulers in the system
+ */
+int lthread_num_schedulers_set(int num);
+
+/**
+ * Return the number of schedulers currently running
+ * @return
+ * the number of schedulers in the system
+ */
+int lthread_active_schedulers(void);
+
+/**
+ * Shutdown the specified scheduler
+ *
+ * This function tells the specified scheduler to
+ * exit if/when there is no more work to do.
+ *
+ * Note that although the scheduler will stop
+ * resources are not freed.
+ *
+ * @param lcore
+ * The lcore of the scheduler to shutdown
+ *
+ * @return
+ * none
+ */
+void lthread_scheduler_shutdown(unsigned lcore);
+
+/**
+ * Shutdown all schedulers
+ *
+ * This function tells all schedulers including the current scheduler to
+ * exit if/when there is no more work to do.
+ *
+ * Note that although the schedulers will stop
+ * resources are not freed.
+ *
+ * @return
+ * none
+ */
+void lthread_scheduler_shutdown_all(void);
+
+/**
+ * Run the lthread scheduler
+ *
+ * Runs the lthread scheduler.
+ * This function returns only if/when all lthreads have exited.
+ * This function must be the main loop of an EAL thread.
+ *
+ * @return
+ * none
+ */
+
+void lthread_run(void);
+
+/**
+ * Create an lthread
+ *
+ * Creates an lthread and places it in the ready queue on a particular
+ * lcore.
+ *
+ * If no scheduler exists yet on the curret lcore then one is created.
+ *
+ * @param new_lt
+ * Pointer to an lthread pointer that will be initialized
+ * @param lcore
+ * the lcore the thread should be started on or the current clore
+ * -1 the current lcore
+ * 0 - LTHREAD_MAX_LCORES any other lcore
+ * @param lthread_func
+ * Pointer to the function the for the thread to run
+ * @param arg
+ * Pointer to args that will be passed to the thread
+ *
+ * @return
+ * 0 success
+ * EAGAIN no resources available
+ * EINVAL NULL thread or function pointer, or lcore_id out of range
+ */
+int
+lthread_create(struct lthread **new_lt,
+ int lcore, lthread_func_t func, void *arg);
+
+/**
+ * Cancel an lthread
+ *
+ * Cancels an lthread and causes it to be terminated
+ * If the lthread is detached it will be freed immediately
+ * otherwise its resources will not be released until it is joined.
+ *
+ * @param new_lt
+ * Pointer to an lthread that will be cancelled
+ *
+ * @return
+ * 0 success
+ * EINVAL thread was NULL
+ */
+int lthread_cancel(struct lthread *lt);
+
+/**
+ * Join an lthread
+ *
+ * Joins the current thread with the specified lthread, and waits for that
+ * thread to exit.
+ * Passes an optional pointer to collect returned data.
+ *
+ * @param lt
+ * Pointer to the lthread to be joined
+ * @param ptr
+ * Pointer to pointer to collect returned data
+ *
+0 * @return
+ * 0 success
+ * EINVAL lthread could not be joined.
+ */
+int lthread_join(struct lthread *lt, void **ptr);
+
+/**
+ * Detach an lthread
+ *
+ * Detaches the current thread
+ * On exit a detached lthread will be freed immediately and will not wait
+ * to be joined. The default state for a thread is not detached.
+ *
+ * @return
+ * none
+ */
+void lthread_detach(void);
+
+/**
+ * Exit an lthread
+ *
+ * Terminate the current thread, optionally return data.
+ * The data may be collected by lthread_join()
+ *
+ * After calling this function the lthread will be suspended until it is
+ * joined. After it is joined then its resources will be freed.
+ *
+ * @param ptr
+ * Pointer to pointer to data to be returned
+ *
+ * @return
+ * none
+ */
+void lthread_exit(void *val);
+
+/**
+ * Cause the current lthread to sleep for n nanoseconds
+ *
+ * The current thread will be suspended until the specified time has elapsed
+ * or has been exceeded.
+ *
+ * Execution will switch to the next lthread that is ready to run
+ *
+ * @param nsecs
+ * Number of nanoseconds to sleep
+ *
+ * @return
+ * none
+ */
+void lthread_sleep(uint64_t nsecs);
+
+/**
+ * Cause the current lthread to sleep for n cpu clock ticks
+ *
+ * The current thread will be suspended until the specified time has elapsed
+ * or has been exceeded.
+ *
+ * Execution will switch to the next lthread that is ready to run
+ *
+ * @param clks
+ * Number of clock ticks to sleep
+ *
+ * @return
+ * none
+ */
+void lthread_sleep_clks(uint64_t clks);
+
+/**
+ * Yield the current lthread
+ *
+ * The current thread will yield and execution will switch to the
+ * next lthread that is ready to run
+ *
+ * @return
+ * none
+ */
+void lthread_yield(void);
+
+/**
+ * Migrate the current thread to another scheduler
+ *
+ * This function migrates the current thread to another scheduler.
+ * Execution will switch to the next lthread that is ready to run on the
+ * current scheduler. The current thread will be resumed on the new scheduler.
+ *
+ * @param lcore
+ * The lcore to migrate to
+ *
+ * @return
+ * 0 success we are now running on the specified core
+ * EINVAL the destination lcore was not valid
+ */
+int lthread_set_affinity(unsigned lcore);
+
+/**
+ * Return the current lthread
+ *
+ * Returns the current lthread
+ *
+ * @return
+ * pointer to the current lthread
+ */
+struct lthread
+*lthread_current(void);
+
+/**
+ * Associate user data with an lthread
+ *
+ * This function sets a user data pointer in the current lthread
+ * The pointer can be retrieved with lthread_get_data()
+ * It is the users responsibility to allocate and free any data referenced
+ * by the user pointer.
+ *
+ * @param data
+ * pointer to user data
+ *
+ * @return
+ * none
+ */
+void lthread_set_data(void *data);
+
+/**
+ * Get user data for the current lthread
+ *
+ * This function returns a user data pointer for the current lthread
+ * The pointer must first be set with lthread_set_data()
+ * It is the users responsibility to allocate and free any data referenced
+ * by the user pointer.
+ *
+ * @return
+ * pointer to user data
+ */
+void
+*lthread_get_data(void);
+
+struct lthread_key;
+typedef void (*tls_destructor_func) (void *);
+
+/**
+ * Create a key for lthread TLS
+ *
+ * This function is modelled on pthread_key_create
+ * It creates a thread-specific data key visible to all lthreads on the
+ * current scheduler.
+ *
+ * Key values may be used to locate thread-specific data.
+ * The same key value may be used by different threads, the values bound
+ * to the key by lthread_setspecific() are maintained on a per-thread
+ * basis and persist for the life of the calling thread.
+ *
+ * An optional destructor function may be associated with each key value.
+ * At thread exit, if a key value has a non-NULL destructor pointer, and the
+ * thread has a non-NULL value associated with the key, the function pointed
+ * to is called with the current associated value as its sole argument.
+ *
+ * @param key
+ * Pointer to the key to be created
+ * @param destructor
+ * Pointer to destructor function
+ *
+ * @return
+ * 0 success
+ * EINVAL the key ptr was NULL
+ * EAGAIN no resources available
+ */
+int lthread_key_create(unsigned int *key, tls_destructor_func destructor);
+
+/**
+ * Delete key for lthread TLS
+ *
+ * This function is modelled on pthread_key_delete().
+ * It deletes a thread-specific data key previously returned by
+ * lthread_key_create().
+ * The thread-specific data values associated with the key need not be NULL
+ * at the time that lthread_key_delete is called.
+ * It is the responsibility of the application to free any application
+ * storage or perform any cleanup actions for data structures related to the
+ * deleted key. This cleanup can be done either before or after
+ * lthread_key_delete is called.
+ *
+ * @param key
+ * The key to be deleted
+ *
+ * @return
+ * 0 Success
+ * EINVAL the key was invalid
+ */
+int lthread_key_delete(unsigned int key);
+
+/**
+ * Get lthread TLS
+ *
+ * This function is modelled on pthread_get_specific().
+ * It returns the value currently bound to the specified key on behalf of the
+ * calling thread. Calling lthread_getspecific() with a key value not
+ * obtained from lthread_key_create() or after key has been deleted with
+ * lthread_key_delete() will result in undefined behaviour.
+ * lthread_getspecific() may be called from a thread-specific data destructor
+ * function.
+ *
+ * @param key
+ * The key for which data is requested
+ *
+ * @return
+ * Pointer to the thread specific data associated with that key
+ * or NULL if no data has been set.
+ */
+void
+*lthread_getspecific(unsigned int key);
+
+/**
+ * Set lthread TLS
+ *
+ * This function is modelled on pthread_set_sepcific()
+ * It associates a thread-specific value with a key obtained via a previous
+ * call to lthread_key_create().
+ * Different threads may bind different values to the same key. These values
+ * are typically pointers to dynamically allocated memory that have been
+ * reserved by the calling thread. Calling lthread_setspecific with a key
+ * value not obtained from lthread_key_create or after the key has been
+ * deleted with lthread_key_delete will result in undefined behaviour.
+ *
+ * @param key
+ * The key for which data is to be set
+ * @param key
+ * Pointer to the user data
+ *
+ * @return
+ * 0 success
+ * EINVAL the key was invalid
+ */
+
+int lthread_setspecific(unsigned int key, const void *value);
+
+/**
+ * The macros below provide an alternative mechanism to access lthread local
+ * storage.
+ *
+ * The macros can be used to declare define and access per lthread local
+ * storage in a similar way to the RTE_PER_LCORE macros which control storage
+ * local to an lcore.
+ *
+ * Memory for per lthread variables declared in this way is allocated when the
+ * lthread is created and a pointer to this memory is stored in the lthread.
+ * The per lthread variables are accessed via the pointer + the offset of the
+ * particular variable.
+ *
+ * The total size of per lthread storage, and the variable offsets are found by
+ * defining the variables in a unique global memory section, the start and end
+ * of which is known. This global memory section is used only in the
+ * computation of the addresses of the lthread variables, and is never actually
+ * used to store any data.
+ *
+ * Due to the fact that variables declared this way may be scattered across
+ * many files, the start and end of the section and variable offsets are only
+ * known after linking, thus the computation of section size and variable
+ * addresses is performed at run time.
+ *
+ * These macros are primarily provided to aid porting of code that makes use
+ * of the existing RTE_PER_LCORE macros. In principle it would be more efficient
+ * to gather all lthread local variables into a single structure and
+ * set/retrieve a pointer to that struct using the alternative
+ * lthread_data_set/get APIs.
+ *
+ * These macros are mutually exclusive with the lthread_data_set/get APIs.
+ * If you define storage using these macros then the lthread_data_set/get APIs
+ * will not perform as expected, the lthread_data_set API does nothing, and the
+ * lthread_data_get API returns the start of global section.
+ *
+ */
+/* start and end of per lthread section */
+extern char __start_per_lt;
+extern char __stop_per_lt;
+
+
+#define RTE_DEFINE_PER_LTHREAD(type, name) \
+__typeof__(type)__attribute((section("per_lt"))) per_lt_##name
+
+/**
+ * Macro to declare an extern per lthread variable "var" of type "type"
+ */
+#define RTE_DECLARE_PER_LTHREAD(type, name) \
+extern __typeof__(type)__attribute((section("per_lt"))) per_lt_##name
+
+/**
+ * Read/write the per-lcore variable value
+ */
+#define RTE_PER_LTHREAD(name) ((typeof(per_lt_##name) *)\
+((char *)lthread_get_data() +\
+((char *) &per_lt_##name - &__start_per_lt)))
+
+/**
+ * Initialize a mutex
+ *
+ * This function provides a mutual exclusion device, the need for which
+ * can normally be avoided in a cooperative multitasking environment.
+ * It is provided to aid porting of legacy code originally written for
+ * preemptive multitasking environments such as pthreads.
+ *
+ * A mutex may be unlocked (not owned by any thread), or locked (owned by
+ * one thread).
+ *
+ * A mutex can never be owned by more than one thread simultaneously.
+ * A thread attempting to lock a mutex that is already locked by another
+ * thread is suspended until the owning thread unlocks the mutex.
+ *
+ * lthread_mutex_init() initializes the mutex object pointed to by mutex
+ * Optional mutex attributes specified in mutexattr, are reserved for future
+ * use and are currently ignored.
+ *
+ * If a thread calls lthread_mutex_lock() on the mutex, then if the mutex
+ * is currently unlocked, it becomes locked and owned by the calling
+ * thread, and lthread_mutex_lock returns immediately. If the mutex is
+ * already locked by another thread, lthread_mutex_lock suspends the calling
+ * thread until the mutex is unlocked.
+ *
+ * lthread_mutex_trylock behaves identically to rte_thread_mutex_lock, except
+ * that it does not block the calling thread if the mutex is already locked
+ * by another thread.
+ *
+ * lthread_mutex_unlock() unlocks the specified mutex. The mutex is assumed
+ * to be locked and owned by the calling thread.
+ *
+ * lthread_mutex_destroy() destroys a mutex object, freeing its resources.
+ * The mutex must be unlocked with nothing blocked on it before calling
+ * lthread_mutex_destroy.
+ *
+ * @param name
+ * Optional pointer to string describing the mutex
+ * @param mutex
+ * Pointer to pointer to the mutex to be initialized
+ * @param attribute
+ * Pointer to attribute - unused reserved
+ *
+ * @return
+ * 0 success
+ * EINVAL mutex was not a valid pointer
+ * EAGAIN insufficient resources
+ */
+
+int
+lthread_mutex_init(char *name, struct lthread_mutex **mutex,
+ const struct lthread_mutexattr *attr);
+
+/**
+ * Destroy a mutex
+ *
+ * This function destroys the specified mutex freeing its resources.
+ * The mutex must be unlocked before calling lthread_mutex_destroy.
+ *
+ * @see lthread_mutex_init()
+ *
+ * @param mutex
+ * Pointer to pointer to the mutex to be initialized
+ *
+ * @return
+ * 0 success
+ * EINVAL mutex was not an initialized mutex
+ * EBUSY mutex was still in use
+ */
+int lthread_mutex_destroy(struct lthread_mutex *mutex);
+
+/**
+ * Lock a mutex
+ *
+ * This function attempts to lock a mutex.
+ * If a thread calls lthread_mutex_lock() on the mutex, then if the mutex
+ * is currently unlocked, it becomes locked and owned by the calling
+ * thread, and lthread_mutex_lock returns immediately. If the mutex is
+ * already locked by another thread, lthread_mutex_lock suspends the calling
+ * thread until the mutex is unlocked.
+ *
+ * @see lthread_mutex_init()
+ *
+ * @param mutex
+ * Pointer to pointer to the mutex to be initialized
+ *
+ * @return
+ * 0 success
+ * EINVAL mutex was not an initialized mutex
+ * EDEADLOCK the mutex was already owned by the calling thread
+ */
+
+int lthread_mutex_lock(struct lthread_mutex *mutex);
+
+/**
+ * Try to lock a mutex
+ *
+ * This function attempts to lock a mutex.
+ * lthread_mutex_trylock behaves identically to rte_thread_mutex_lock, except
+ * that it does not block the calling thread if the mutex is already locked
+ * by another thread.
+ *
+ *
+ * @see lthread_mutex_init()
+ *
+ * @param mutex
+ * Pointer to pointer to the mutex to be initialized
+ *
+ * @return
+ * 0 success
+ * EINVAL mutex was not an initialized mutex
+ * EBUSY the mutex was already locked by another thread
+ */
+int lthread_mutex_trylock(struct lthread_mutex *mutex);
+
+/**
+ * Unlock a mutex
+ *
+ * This function attempts to unlock the specified mutex. The mutex is assumed
+ * to be locked and owned by the calling thread.
+ *
+ * The oldest of any threads blocked on the mutex is made ready and may
+ * compete with any other running thread to gain the mutex, it fails it will
+ * be blocked again.
+ *
+ * @param mutex
+ * Pointer to pointer to the mutex to be initialized
+ *
+ * @return
+ * 0 mutex was unlocked
+ * EINVAL mutex was not an initialized mutex
+ * EPERM the mutex was not owned by the calling thread
+ */
+
+int lthread_mutex_unlock(struct lthread_mutex *mutex);
+
+/**
+ * Initialize a condition variable
+ *
+ * This function initializes a condition variable.
+ *
+ * Condition variables can be used to communicate changes in the state of data
+ * shared between threads.
+ *
+ * @see lthread_cond_wait()
+ *
+ * @param name
+ * Pointer to optional string describing the condition variable
+ * @param c
+ * Pointer to pointer to the condition variable to be initialized
+ * @param attr
+ * Pointer to optional attribute reserved for future use, currently ignored
+ *
+ * @return
+ * 0 success
+ * EINVAL cond was not a valid pointer
+ * EAGAIN insufficient resources
+ */
+int
+lthread_cond_init(char *name, struct lthread_cond **c,
+ const struct lthread_condattr *attr);
+
+/**
+ * Destroy a condition variable
+ *
+ * This function destroys a condition variable that was created with
+ * lthread_cond_init() and releases its resources.
+ *
+ * @param cond
+ * Pointer to pointer to the condition variable to be destroyed
+ *
+ * @return
+ * 0 Success
+ * EBUSY condition variable was still in use
+ * EINVAL was not an initialised condition variable
+ */
+int lthread_cond_destroy(struct lthread_cond *cond);
+
+/**
+ * Wait on a condition variable
+ *
+ * The function blocks the current thread waiting on the condition variable
+ * specified by cond. The waiting thread unblocks only after another thread
+ * calls lthread_cond_signal, or lthread_cond_broadcast, specifying the
+ * same condition variable.
+ *
+ * @param cond
+ * Pointer to pointer to the condition variable to be waited on
+ *
+ * @param reserved
+ * reserved for future use
+ *
+ * @return
+ * 0 The condition was signalled ( Success )
+ * EINVAL was not a an initialised condition variable
+ */
+int lthread_cond_wait(struct lthread_cond *c, uint64_t reserved);
+
+/**
+ * Signal a condition variable
+ *
+ * The function unblocks one thread waiting for the condition variable cond.
+ * If no threads are waiting on cond, the rte_lthead_cond_signal() function
+ * has no effect.
+ *
+ * @param cond
+ * Pointer to pointer to the condition variable to be signalled
+ *
+ * @return
+ * 0 The condition was signalled ( Success )
+ * EINVAL was not a an initialised condition variable
+ */
+int lthread_cond_signal(struct lthread_cond *c);
+
+/**
+ * Broadcast a condition variable
+ *
+ * The function unblocks all threads waiting for the condition variable cond.
+ * If no threads are waiting on cond, the rte_lthead_cond_broadcast()
+ * function has no effect.
+ *
+ * @param cond
+ * Pointer to pointer to the condition variable to be signalled
+ *
+ * @return
+ * 0 The condition was signalled ( Success )
+ * EINVAL was not a an initialised condition variable
+ */
+int lthread_cond_broadcast(struct lthread_cond *c);
+
+#endif /* LTHREAD_H */
diff --git a/examples/performance-thread/common/lthread_cond.c b/examples/performance-thread/common/lthread_cond.c
new file mode 100644
index 0000000..cfd540b
--- /dev/null
+++ b/examples/performance-thread/common/lthread_cond.c
@@ -0,0 +1,240 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Some portions of this software may have been derived from the
+ * https://github.com/halayli/lthread which carrys the following license.
+ *
+ * Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <limits.h>
+#include <inttypes.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <fcntl.h>
+#include <sys/time.h>
+#include <sys/mman.h>
+#include <errno.h>
+
+#include <rte_config.h>
+#include <rte_log.h>
+#include <rte_common.h>
+
+#include "lthread_api.h"
+#include "lthread_diag_api.h"
+#include "lthread_diag.h"
+#include "lthread_int.h"
+#include "lthread_sched.h"
+#include "lthread_queue.h"
+#include "lthread_objcache.h"
+#include "lthread_timer.h"
+#include "lthread_mutex.h"
+#include "lthread_cond.h"
+
+/*
+ * Create a condition variable
+ */
+int
+lthread_cond_init(char *name, struct lthread_cond **cond,
+ __rte_unused const struct lthread_condattr *attr)
+{
+ struct lthread_cond *c;
+
+ if (cond == NULL)
+ return POSIX_ERRNO(EINVAL);
+
+ /* allocate a condition variable from cache */
+ c = _lthread_objcache_alloc((THIS_SCHED)->cond_cache);
+
+ if (c == NULL)
+ return POSIX_ERRNO(EAGAIN);
+
+ c->blocked = _lthread_queue_create("blocked");
+ if (c->blocked == NULL) {
+ _lthread_objcache_free((THIS_SCHED)->cond_cache, (void *)c);
+ return POSIX_ERRNO(EAGAIN);
+ }
+
+ if (name == NULL)
+ strncpy(c->name, "no name", sizeof(c->name));
+ else
+ strncpy(c->name, name, sizeof(c->name));
+ c->name[sizeof(c->name)-1] = 0;
+
+ c->root_sched = THIS_SCHED;
+
+ (*cond) = c;
+ DIAG_CREATE_EVENT((*cond), LT_DIAG_COND_CREATE);
+ return 0;
+}
+
+/*
+ * Destroy a condition variable
+ */
+int lthread_cond_destroy(struct lthread_cond *c)
+{
+ if (c == NULL) {
+ DIAG_EVENT(c, LT_DIAG_COND_DESTROY, c, POSIX_ERRNO(EINVAL));
+ return POSIX_ERRNO(EINVAL);
+ }
+
+ /* try to free it */
+ if (_lthread_queue_destroy(c->blocked) < 0) {
+ /* queue in use */
+ DIAG_EVENT(c, LT_DIAG_COND_DESTROY, c, POSIX_ERRNO(EBUSY));
+ return POSIX_ERRNO(EBUSY);
+ }
+
+ /* okay free it */
+ _lthread_objcache_free(c->root_sched->cond_cache, c);
+ DIAG_EVENT(c, LT_DIAG_COND_DESTROY, c, 0);
+ return 0;
+}
+
+/*
+ * Wait on a condition variable
+ */
+int lthread_cond_wait(struct lthread_cond *c, __rte_unused uint64_t reserved)
+{
+ struct lthread *lt = THIS_LTHREAD;
+
+ if (c == NULL) {
+ DIAG_EVENT(c, LT_DIAG_COND_WAIT, c, POSIX_ERRNO(EINVAL));
+ return POSIX_ERRNO(EINVAL);
+ }
+
+
+ DIAG_EVENT(c, LT_DIAG_COND_WAIT, c, 0);
+
+ /* queue the current thread in the blocked queue
+ * this will be written when we return to the scheduler
+ * to ensure that the current thread context is saved
+ * before any signal could result in it being dequeued and
+ * resumed
+ */
+ lt->pending_wr_queue = c->blocked;
+ _suspend();
+
+ /* the condition happened */
+ return 0;
+}
+
+/*
+ * Signal a condition variable
+ * attempt to resume any blocked thread
+ */
+int lthread_cond_signal(struct lthread_cond *c)
+{
+ struct lthread *lt;
+
+ if (c == NULL) {
+ DIAG_EVENT(c, LT_DIAG_COND_SIGNAL, c, POSIX_ERRNO(EINVAL));
+ return POSIX_ERRNO(EINVAL);
+ }
+
+ lt = _lthread_queue_remove(c->blocked);
+
+ if (lt != NULL) {
+ /* okay wake up this thread */
+ DIAG_EVENT(c, LT_DIAG_COND_SIGNAL, c, lt);
+ _ready_queue_insert((struct lthread_sched *)lt->sched, lt);
+ }
+ return 0;
+}
+
+/*
+ * Broadcast a condition variable
+ */
+int lthread_cond_broadcast(struct lthread_cond *c)
+{
+ struct lthread *lt;
+
+ if (c == NULL) {
+ DIAG_EVENT(c, LT_DIAG_COND_BROADCAST, c, POSIX_ERRNO(EINVAL));
+ return POSIX_ERRNO(EINVAL);
+ }
+
+ DIAG_EVENT(c, LT_DIAG_COND_BROADCAST, c, 0);
+ do {
+ /* drain the queue waking everybody */
+ lt = _lthread_queue_remove(c->blocked);
+
+ if (lt != NULL) {
+ DIAG_EVENT(c, LT_DIAG_COND_BROADCAST, c, lt);
+ /* wake up */
+ _ready_queue_insert((struct lthread_sched *)lt->sched,
+ lt);
+ }
+ } while (!_lthread_queue_empty(c->blocked));
+ _reschedule();
+ DIAG_EVENT(c, LT_DIAG_COND_BROADCAST, c, 0);
+ return 0;
+}
+
+/*
+ * return the diagnostic ref val stored in a condition var
+ */
+uint64_t
+lthread_cond_diag_ref(struct lthread_cond *c)
+{
+ if (c == NULL)
+ return 0;
+ return c->diag_ref;
+}
diff --git a/examples/performance-thread/common/lthread_cond.h b/examples/performance-thread/common/lthread_cond.h
new file mode 100644
index 0000000..5bd02a7
--- /dev/null
+++ b/examples/performance-thread/common/lthread_cond.h
@@ -0,0 +1,77 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Some portions of this software may have been derived from the
+ * https://github.com/halayli/lthread which carrys the following license.
+ *
+ * Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef LTHREAD_COND_H_
+#define LTHREAD_COND_H_
+
+#include "lthread_queue.h"
+
+#define MAX_COND_NAME_SIZE 64
+
+struct lthread_cond {
+ struct lthread_queue *blocked;
+ struct lthread_sched *root_sched;
+ int count;
+ char name[MAX_COND_NAME_SIZE];
+ uint64_t diag_ref; /* optional ref to user diag data */
+} __rte_cache_aligned;
+
+#endif /* LTHREAD_COND_H_ */
diff --git a/examples/performance-thread/common/lthread_diag.c b/examples/performance-thread/common/lthread_diag.c
new file mode 100644
index 0000000..dddc6f0
--- /dev/null
+++ b/examples/performance-thread/common/lthread_diag.c
@@ -0,0 +1,321 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <rte_config.h>
+#include <rte_log.h>
+#include <rte_common.h>
+
+#include "lthread_diag.h"
+#include "lthread_queue.h"
+#include "lthread_pool.h"
+#include "lthread_objcache.h"
+#include "lthread_sched.h"
+#include "lthread_diag_api.h"
+
+
+/* dummy ref value of default diagnostic callback */
+static uint64_t dummy_ref;
+
+#define DIAG_SCHED_STATS_FORMAT \
+"core %d\n%33s %12s %12s %12s %12s\n"
+
+#define DIAG_CACHE_STATS_FORMAT \
+"%20s %12lu %12lu %12lu %12lu %12lu\n"
+
+#define DIAG_QUEUE_STATS_FORMAT \
+"%20s %12lu %12lu %12lu\n"
+
+
+/*
+ * texts used in diagnostic events,
+ * corresponding diagnostic mask bit positions are given as comment
+ */
+const char *diag_event_text[] = {
+ "LTHREAD_CREATE ", /* 00 */
+ "LTHREAD_EXIT ", /* 01 */
+ "LTHREAD_JOIN ", /* 02 */
+ "LTHREAD_CANCEL ", /* 03 */
+ "LTHREAD_DETACH ", /* 04 */
+ "LTHREAD_FREE ", /* 05 */
+ "LTHREAD_SUSPENDED ", /* 06 */
+ "LTHREAD_YIELD ", /* 07 */
+ "LTHREAD_RESCHEDULED", /* 08 */
+ "LTHREAD_SLEEP ", /* 09 */
+ "LTHREAD_RESUMED ", /* 10 */
+ "LTHREAD_AFFINITY ", /* 11 */
+ "LTHREAD_TMR_START ", /* 12 */
+ "LTHREAD_TMR_DELETE ", /* 13 */
+ "LTHREAD_TMR_EXPIRED", /* 14 */
+ "COND_CREATE ", /* 15 */
+ "COND_DESTROY ", /* 16 */
+ "COND_WAIT ", /* 17 */
+ "COND_SIGNAL ", /* 18 */
+ "COND_BROADCAST ", /* 19 */
+ "MUTEX_CREATE ", /* 20 */
+ "MUTEX_DESTROY ", /* 21 */
+ "MUTEX_LOCK ", /* 22 */
+ "MUTEX_TRYLOCK ", /* 23 */
+ "MUTEX_BLOCKED ", /* 24 */
+ "MUTEX_UNLOCKED ", /* 25 */
+ "SCHED_CREATE ", /* 26 */
+ "SCHED_SHUTDOWN " /* 27 */
+};
+
+
+/*
+ * set diagnostic ,ask
+ */
+void lthread_diagnostic_set_mask(DIAG_USED uint64_t mask)
+{
+#if LTHREAD_DIAG
+ diag_mask = mask;
+#else
+ RTE_LOG(INFO, LTHREAD,
+ "LTHREAD_DIAG is not set, see lthread_diag_api.h\n");
+#endif
+}
+
+
+/*
+ * Check consistency of the scheduler stats
+ * Only sensible run after the schedulers are stopped
+ * Count the number of objects lying in caches and queues
+ * and available in the qnode pool.
+ * This should be equal to the total capacity of all
+ * qnode pools.
+ */
+void
+_sched_stats_consistency_check(void);
+void
+_sched_stats_consistency_check(void)
+{
+#if LTHREAD_DIAG
+ int i;
+ struct lthread_sched *sched;
+ uint64_t count = 0;
+ uint64_t capacity = 0;
+
+ for (i = 0; i < LTHREAD_MAX_LCORES; i++) {
+ sched = schedcore[i];
+ if (sched == NULL)
+ continue;
+
+ /* each of these queues consumes a stub node */
+ count += 8;
+ count += DIAG_COUNT(sched->ready, size);
+ count += DIAG_COUNT(sched->pready, size);
+ count += DIAG_COUNT(sched->lthread_cache, available);
+ count += DIAG_COUNT(sched->stack_cache, available);
+ count += DIAG_COUNT(sched->tls_cache, available);
+ count += DIAG_COUNT(sched->per_lthread_cache, available);
+ count += DIAG_COUNT(sched->cond_cache, available);
+ count += DIAG_COUNT(sched->mutex_cache, available);
+
+ /* the node pool does not consume a stub node */
+ if (sched->qnode_pool->fast_alloc != NULL)
+ count++;
+ count += DIAG_COUNT(sched->qnode_pool, available);
+
+ capacity += DIAG_COUNT(sched->qnode_pool, capacity);
+ }
+ if (count != capacity) {
+ RTE_LOG(CRIT, LTHREAD,
+ "Scheduler caches are inconsistent\n");
+ } else {
+ RTE_LOG(INFO, LTHREAD,
+ "Scheduler caches are ok\n");
+ }
+#endif
+}
+
+/*
+ * Display node pool stats
+ */
+static inline void
+_qnode_pool_display(DIAG_USED struct qnode_pool *p)
+{
+#if LTHREAD_DIAG
+
+ printf(DIAG_CACHE_STATS_FORMAT,
+ p->name,
+ DIAG_COUNT(p, rd),
+ DIAG_COUNT(p, wr),
+ DIAG_COUNT(p, available),
+ DIAG_COUNT(p, prealloc),
+ DIAG_COUNT(p, capacity));
+ fflush(stdout);
+#endif
+}
+
+
+/*
+ * Display queue stats
+ */
+static inline void
+_lthread_queue_display(DIAG_USED struct lthread_queue *q)
+{
+#if LTHREAD_DIAG
+
+ printf(DIAG_QUEUE_STATS_FORMAT,
+ q->name,
+ DIAG_COUNT(q, rd),
+ DIAG_COUNT(q, wr),
+ DIAG_COUNT(q, size));
+ fflush(stdout);
+#endif
+}
+
+/*
+ * Display objcache stats
+ */
+static inline void
+_objcache_display(DIAG_USED struct lthread_objcache *c)
+{
+#if LTHREAD_DIAG
+
+ printf(DIAG_CACHE_STATS_FORMAT,
+ c->name,
+ DIAG_COUNT(c, rd),
+ DIAG_COUNT(c, wr),
+ DIAG_COUNT(c, available),
+ DIAG_COUNT(c, prealloc),
+ DIAG_COUNT(c, capacity));
+#if DISPLAY_OBCACHE_QUEUES
+ _lthread_queue_display(c->q);
+#endif
+ fflush(stdout);
+#endif
+}
+
+
+/*
+ * Display sched stats
+ */
+void
+lthread_sched_stats_display(void)
+{
+#if LTHREAD_DIAG
+ int i;
+ struct lthread_sched *sched;
+
+ for (i = 0; i < LTHREAD_MAX_LCORES; i++) {
+ sched = schedcore[i];
+ if (sched != NULL) {
+ printf(DIAG_SCHED_STATS_FORMAT,
+ sched->lcore_id,
+ "rd",
+ "wr",
+ "present",
+ "nb preallocs",
+ "capacity");
+ _lthread_queue_display(sched->ready);
+ _lthread_queue_display(sched->pready);
+ _qnode_pool_display(sched->qnode_pool);
+ _objcache_display(sched->lthread_cache);
+ _objcache_display(sched->stack_cache);
+ _objcache_display(sched->tls_cache);
+ _objcache_display(sched->per_lthread_cache);
+ _objcache_display(sched->cond_cache);
+ _objcache_display(sched->mutex_cache);
+ fflush(stdout);
+ }
+ }
+ _sched_stats_consistency_check();
+#else
+ RTE_LOG(INFO, LTHREAD,
+ "lthread diagnostics disabled\n"
+ "hint - set LTHREAD_DIAG in lthread_diag_api.h\n");
+#endif
+}
+
+/*
+ * Defafult diagnostic callback
+ */
+static uint64_t
+_lthread_diag_default_cb(uint64_t time, struct lthread *lt, int diag_event,
+ uint64_t diag_ref, const char *text, uint64_t p1, uint64_t p2)
+{
+ uint64_t _p2;
+ int lcore = (int) rte_lcore_id();
+
+ switch (diag_event) {
+ case LT_DIAG_LTHREAD_CREATE:
+ case LT_DIAG_MUTEX_CREATE:
+ case LT_DIAG_COND_CREATE:
+ _p2 = dummy_ref;
+ break;
+ default:
+ _p2 = p2;
+ break;
+ }
+
+ printf("%"PRIu64" %d %8.8lx %8.8lx %s %8.8lx %8.8lx\n",
+ time,
+ lcore,
+ (uint64_t) lt,
+ diag_ref,
+ text,
+ p1,
+ _p2);
+
+ return dummy_ref++;
+}
+
+/*
+ * plug in default diag callback with mask off
+ */
+void _lthread_diag_ctor(void)__attribute__((constructor));
+void _lthread_diag_ctor(void)
+{
+ diag_cb = _lthread_diag_default_cb;
+ diag_mask = 0;
+}
+
+
+/*
+ * enable diagnostics
+ */
+void lthread_diagnostic_enable(DIAG_USED diag_callback cb,
+ DIAG_USED uint64_t mask)
+{
+#if LTHREAD_DIAG
+ if (cb == NULL)
+ diag_cb = _lthread_diag_default_cb;
+ else
+ diag_cb = cb;
+ diag_mask = mask;
+#else
+ RTE_LOG(INFO, LTHREAD,
+ "LTHREAD_DIAG is not set, see lthread_diag_api.h\n");
+#endif
+}
diff --git a/examples/performance-thread/common/lthread_diag.h b/examples/performance-thread/common/lthread_diag.h
new file mode 100644
index 0000000..d08a5b9
--- /dev/null
+++ b/examples/performance-thread/common/lthread_diag.h
@@ -0,0 +1,129 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef LTHREAD_DIAG_H_
+#define LTHREAD_DIAG_H_
+
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <rte_log.h>
+#include <rte_common.h>
+
+#include "lthread_api.h"
+#include "lthread_diag_api.h"
+
+extern diag_callback diag_cb;
+
+extern const char *diag_event_text[];
+extern uint64_t diag_mask;
+
+/* max size of name strings */
+#define LT_MAX_NAME_SIZE 64
+
+#if LTHREAD_DIAG
+
+/*
+ * Generate a diagnostic trace or event in the case where an object is created.
+ *
+ * The value returned by the callback is stored in the object.
+ *
+ * @ param obj
+ * pointer to the object that was created
+ * @ param ev
+ * the event code
+ *
+ */
+#define DIAG_CREATE_EVENT(obj, ev) do { \
+ struct lthread *ct = RTE_PER_LCORE(this_sched)->current_lthread;\
+ if ((BIT(ev) & diag_mask) && (ev < LT_DIAG_EVENT_MAX)) { \
+ (obj)->diag_ref = (diag_cb)(rte_rdtsc(), \
+ ct, \
+ (ev), \
+ 0, \
+ diag_event_text[(ev)], \
+ (uint64_t)obj, \
+ 0); \
+ } \
+} while (0)
+
+/*
+ * Generate a diagnostic trace event.
+ *
+ * @ param obj
+ * pointer to the lthread, cond or mutex object
+ * @ param ev
+ * the event code
+ * @ param p1
+ * object specific value ( see lthread_diag_api.h )
+ * @ param p2
+ * object specific value ( see lthread_diag_api.h )
+ */
+#define DIAG_EVENT(obj, ev, p1, p2) do { \
+ struct lthread *ct = RTE_PER_LCORE(this_sched)->current_lthread;\
+ if ((BIT(ev) & diag_mask) && (ev < LT_DIAG_EVENT_MAX)) { \
+ (diag_cb)(rte_rdtsc(), \
+ ct, \
+ ev, \
+ (obj)->diag_ref, \
+ diag_event_text[(ev)], \
+ (uint64_t)(p1), \
+ (uint64_t)(p2)); \
+ } \
+} while (0)
+
+#define DIAG_COUNT_DEFINE(x) rte_atomic64_t count_##x
+#define DIAG_COUNT_INIT(o, x) rte_atomic64_init(&((o)->count_##x))
+#define DIAG_COUNT_INC(o, x) rte_atomic64_inc(&((o)->count_##x))
+#define DIAG_COUNT_DEC(o, x) rte_atomic64_dec(&((o)->count_##x))
+#define DIAG_COUNT(o, x) rte_atomic64_read(&((o)->count_##x))
+
+#define DIAG_USED
+
+#else
+
+/* no diagnostics configured */
+
+#define DIAG_CREATE_EVENT(obj, ev)
+#define DIAG_EVENT(obj, ev, p1, p)
+
+#define DIAG_COUNT_DEFINE(x)
+#define DIAG_COUNT_INIT(o, x) do {} while (0)
+#define DIAG_COUNT_INC(o, x) do {} while (0)
+#define DIAG_COUNT_DEC(o, x) do {} while (0)
+#define DIAG_COUNT(o, x) 0
+
+#define DIAG_USED __rte_unused
+
+#endif /* LTHREAD_DIAG */
+#endif /* LTHREAD_DIAG_H_ */
diff --git a/examples/performance-thread/common/lthread_diag_api.h b/examples/performance-thread/common/lthread_diag_api.h
new file mode 100644
index 0000000..d8e477b
--- /dev/null
+++ b/examples/performance-thread/common/lthread_diag_api.h
@@ -0,0 +1,319 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef LTHREAD_DIAG_API_H_
+#define LTHREAD_DIAG_API_H_
+
+#include <stdint.h>
+#include <inttypes.h>
+
+/*
+ * Enable diagnostics
+ * 0 = conditionally compiled out
+ * 1 = compiled in and maskable at run time, see below for details
+ */
+#define LTHREAD_DIAG 0
+
+/**
+ * lthread diagnostic interface
+ *
+ * If enabled via configuration file option ( tbd ) the lthread subsystem
+ * can generate selected trace information, either RTE_LOG (INFO) messages,
+ * or else invoke a user supplied callback function when any of the events
+ * listed below occur.
+ *
+ * Reporting of events can be selectively masked, the bit position in the
+ * mask is determined by the corresponding event identifier listed below.
+ *
+ * Diagnostics are enabled by registering the callback function and mask
+ * using the API lthread_diagnostic_enable().
+ *
+ * Various interesting parameters are passed to the callback, including the
+ * time in cpu clks, the lthread id, the diagnostic event id, a user ref value,
+ * event text string, object being traced, and two context dependent parameters
+ * (p1 and p2). The meaning of the two parameters p1 and p2 depends on
+ * the specific event.
+ *
+ * The events LT_DIAG_LTHREAD_CREATE, LT_DIAG_MUTEX_CREATE and
+ * LT_DIAG_COND_CREATE are implicitly enabled if the event mask includes any of
+ * the LT_DIAG_LTHREAD_XXX, LT_DIAG_MUTEX_XXX or LT_DIAG_COND_XXX events
+ * respectively.
+ *
+ * These create events may also be included in the mask discreetly if it is
+ * desired to monitor only create events.
+ *
+ * @param time
+ * The time in cpu clks at which the event occurred
+ *
+ * @param lthread
+ * The current lthread
+ *
+ * @param diag_event
+ * The diagnostic event id (bit position in the mask)
+ *
+ * @param diag_ref
+ *
+ * For LT_DIAG_LTHREAD_CREATE, LT_DIAG_MUTEX_CREATE or LT_DIAG_COND_CREATE
+ * this parameter is not used and set to 0.
+ * All other events diag_ref contains the user ref value returned by the
+ * callback function when lthread is created.
+ *
+ * The diag_ref values assigned to mutex and cond var can be retrieved
+ * using the APIs lthread_mutex_diag_ref(), and lthread_cond_diag_ref()
+ * respectively.
+ *
+ * @param p1
+ * see below
+ *
+ * @param p1
+ * see below
+ *
+ * @returns
+ * For LT_DIAG_LTHREAD_CREATE, LT_DIAG_MUTEX_CREATE or LT_DIAG_COND_CREATE
+ * expects a user diagnostic ref value that will be saved in the lthread, mutex
+ * or cond var.
+ *
+ * For all other events return value is ignored.
+ *
+ * LT_DIAG_SCHED_CREATE - Invoked when a scheduler is created
+ * p1 = the scheduler that was created
+ * p2 = not used
+ * return value will be ignored
+ *
+ * LT_DIAG_SCHED_SHUTDOWN - Invoked when a shutdown request is received
+ * p1 = the scheduler to be shutdown
+ * p2 = not used
+ * return value will be ignored
+ *
+ * LT_DIAG_LTHREAD_CREATE - Invoked when a thread is created
+ * p1 = the lthread that was created
+ * p2 = not used
+ * return value will be stored in the lthread
+ *
+ * LT_DIAG_LTHREAD_EXIT - Invoked when a lthread exits
+ * p2 = 0 if the thread was already joined
+ * p2 = 1 if the thread was not already joined
+ * return val ignored
+ *
+ * LT_DIAG_LTHREAD_JOIN - Invoked when a lthread exits
+ * p1 = the lthread that is being joined
+ * p2 = 0 if the thread was already exited
+ * p2 = 1 if the thread was not already exited
+ * return val ignored
+ *
+ * LT_DIAG_LTHREAD_CANCELLED - Invoked when an lthread is cancelled
+ * p1 = not used
+ * p2 = not used
+ * return val ignored
+ *
+ * LT_DIAG_LTHREAD_DETACH - Invoked when an lthread is detached
+ * p1 = not used
+ * p2 = not used
+ * return val ignored
+ *
+ * LT_DIAG_LTHREAD_FREE - Invoked when an lthread is freed
+ * p1 = not used
+ * p2 = not used
+ * return val ignored
+ *
+ * LT_DIAG_LTHREAD_SUSPENDED - Invoked when an lthread is suspended
+ * p1 = not used
+ * p2 = not used
+ * return val ignored
+ *
+ * LT_DIAG_LTHREAD_YIELD - Invoked when an lthread explicitly yields
+ * p1 = not used
+ * p2 = not used
+ * return val ignored
+ *
+ * LT_DIAG_LTHREAD_RESCHEDULED - Invoked when an lthread is rescheduled
+ * p1 = not used
+ * p2 = not used
+ * return val ignored
+ *
+ * LT_DIAG_LTHREAD_RESUMED - Invoked when an lthread is resumed
+ * p1 = not used
+ * p2 = not used
+ * return val ignored
+ *
+ * LT_DIAG_LTHREAD_AFFINITY - Invoked when an lthread is affinitised
+ * p1 = the destination lcore_id
+ * p2 = not used
+ * return val ignored
+ *
+ * LT_DIAG_LTHREAD_TMR_START - Invoked when an lthread starts a timer
+ * p1 = address of timer node
+ * p2 = the timeout value
+ * return val ignored
+ *
+ * LT_DIAG_LTHREAD_TMR_DELETE - Invoked when an lthread deletes a timer
+ * p1 = address of the timer node
+ * p2 = 0 the timer and the was successfully deleted
+ * p2 = not usee
+ * return val ignored
+ *
+ * LT_DIAG_LTHREAD_TMR_EXPIRED - Invoked when an lthread timer expires
+ * p1 = address of scheduler the timer expired on
+ * p2 = the thread associated with the timer
+ * return val ignored
+ *
+ * LT_DIAG_COND_CREATE - Invoked when a condition variable is created
+ * p1 = address of cond var that was created
+ * p2 = not used
+ * return diag ref value will be stored in the condition variable
+ *
+ * LT_DIAG_COND_DESTROY - Invoked when a condition variable is destroyed
+ * p1 = not used
+ * p2 = not used
+ * return val ignored
+ *
+ * LT_DIAG_COND_WAIT - Invoked when an lthread waits on a cond var
+ * p1 = the address of the condition variable
+ * p2 = not used
+ * return val ignored
+ *
+ * LT_DIAG_COND_SIGNAL - Invoked when an lthread signals a cond var
+ * p1 = the address of the cond var
+ * p2 = the lthread that was signalled, or error code
+ * return val ignored
+ *
+ * LT_DIAG_COND_BROADCAST - Invoked when an lthread broadcasts a cond var
+ * p1 = the address of the condition variable
+ * p2 = the lthread(s) that are signalled, or error code
+ *
+ * LT_DIAG_MUTEX_CREATE - Invoked when a mutex is created
+ * p1 = address of muex
+ * p2 = not used
+ * return diag ref value will be stored in the mutex variable
+ *
+ * LT_DIAG_MUTEX_DESTROY - Invoked when a mutex is destroyed
+ * p1 = address of mutex
+ * p2 = not used
+ * return val ignored
+ *
+ * LT_DIAG_MUTEX_LOCK - Invoked when a mutex lock is obtained
+ * p1 = address of mutex
+ * p2 = function return value
+ * return val ignored
+ *
+ * LT_DIAG_MUTEX_BLOCKED - Invoked when an lthread blocks on a mutex
+ * p1 = address of mutex
+ * p2 = function return value
+ * return val ignored
+ *
+ * LT_DIAG_MUTEX_TRYLOCK - Invoked when a mutex try lock is attempted
+ * p1 = address of mutex
+ * p2 = the function return value
+ * return val ignored
+ *
+ * LT_DIAG_MUTEX_UNLOCKED - Invoked when a mutex is unlocked
+ * p1 = address of mutex
+ * p2 = the thread that was unlocked, or error code
+ * return val ignored
+ */
+typedef uint64_t (*diag_callback) (uint64_t time, struct lthread *lt,
+ int diag_event, uint64_t diag_ref,
+ const char *text, uint64_t p1, uint64_t p2);
+
+/*
+ * Set user diagnostic callback and mask
+ * If the callback function pointer is NULL the default
+ * callback handler will be restored.
+ */
+void lthread_diagnostic_enable(diag_callback cb, uint64_t diag_mask);
+
+/*
+ * Set diagnostic mask
+ */
+void lthread_diagnostic_set_mask(uint64_t mask);
+
+/*
+ * lthread diagnostic callback
+ */
+enum lthread_diag_ev {
+ /* bits 0 - 14 lthread flag group */
+ LT_DIAG_LTHREAD_CREATE, /* 00 mask 0x00000001 */
+ LT_DIAG_LTHREAD_EXIT, /* 01 mask 0x00000002 */
+ LT_DIAG_LTHREAD_JOIN, /* 02 mask 0x00000004 */
+ LT_DIAG_LTHREAD_CANCEL, /* 03 mask 0x00000008 */
+ LT_DIAG_LTHREAD_DETACH, /* 04 mask 0x00000010 */
+ LT_DIAG_LTHREAD_FREE, /* 05 mask 0x00000020 */
+ LT_DIAG_LTHREAD_SUSPENDED, /* 06 mask 0x00000040 */
+ LT_DIAG_LTHREAD_YIELD, /* 07 mask 0x00000080 */
+ LT_DIAG_LTHREAD_RESCHEDULED, /* 08 mask 0x00000100 */
+ LT_DIAG_LTHREAD_SLEEP, /* 09 mask 0x00000200 */
+ LT_DIAG_LTHREAD_RESUMED, /* 10 mask 0x00000400 */
+ LT_DIAG_LTHREAD_AFFINITY, /* 11 mask 0x00000800 */
+ LT_DIAG_LTHREAD_TMR_START, /* 12 mask 0x00001000 */
+ LT_DIAG_LTHREAD_TMR_DELETE, /* 13 mask 0x00002000 */
+ LT_DIAG_LTHREAD_TMR_EXPIRED, /* 14 mask 0x00004000 */
+ /* bits 15 - 19 conditional variable flag group */
+ LT_DIAG_COND_CREATE, /* 15 mask 0x00008000 */
+ LT_DIAG_COND_DESTROY, /* 16 mask 0x00010000 */
+ LT_DIAG_COND_WAIT, /* 17 mask 0x00020000 */
+ LT_DIAG_COND_SIGNAL, /* 18 mask 0x00040000 */
+ LT_DIAG_COND_BROADCAST, /* 19 mask 0x00080000 */
+ /* bits 20 - 25 mutex flag group */
+ LT_DIAG_MUTEX_CREATE, /* 20 mask 0x00100000 */
+ LT_DIAG_MUTEX_DESTROY, /* 21 mask 0x00200000 */
+ LT_DIAG_MUTEX_LOCK, /* 22 mask 0x00400000 */
+ LT_DIAG_MUTEX_TRYLOCK, /* 23 mask 0x00800000 */
+ LT_DIAG_MUTEX_BLOCKED, /* 24 mask 0x01000000 */
+ LT_DIAG_MUTEX_UNLOCKED, /* 25 mask 0x02000000 */
+ /* bits 26 - 27 scheduler flag group - 8 bits */
+ LT_DIAG_SCHED_CREATE, /* 26 mask 0x04000000 */
+ LT_DIAG_SCHED_SHUTDOWN, /* 27 mask 0x08000000 */
+ LT_DIAG_EVENT_MAX
+};
+
+#define LT_DIAG_ALL 0xffffffffffffffff
+
+
+/*
+ * Display scheduler stats
+ */
+void
+lthread_sched_stats_display(void);
+
+/*
+ * return the diagnostic ref val stored in a condition var
+ */
+uint64_t
+lthread_cond_diag_ref(struct lthread_cond *c);
+
+/*
+ * return the diagnostic ref val stored in a mutex
+ */
+uint64_t
+lthread_mutex_diag_ref(struct lthread_mutex *m);
+
+#endif /* LTHREAD_DIAG_API_H_ */
diff --git a/examples/performance-thread/common/lthread_int.h b/examples/performance-thread/common/lthread_int.h
new file mode 100644
index 0000000..c8357f4
--- /dev/null
+++ b/examples/performance-thread/common/lthread_int.h
@@ -0,0 +1,212 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Some portions of this software may have been derived from the
+ * https://github.com/halayli/lthread which carrys the following license.
+ *
+ * Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+#ifndef LTHREAD_INT_H
+#include <lthread_api.h>
+#define LTHREAD_INT_H
+
+#include <stdint.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <errno.h>
+#include <pthread.h>
+#include <time.h>
+
+#include <rte_cycles.h>
+#include <rte_per_lcore.h>
+#include <rte_timer.h>
+#include <rte_ring.h>
+#include <rte_atomic_64.h>
+#include <rte_spinlock.h>
+#include <ctx.h>
+
+#include <lthread_api.h>
+#include "lthread.h"
+#include "lthread_diag.h"
+#include "lthread_tls.h"
+
+struct lthread;
+struct lthread_sched;
+struct lthread_cond;
+struct lthread_mutex;
+struct lthread_key;
+
+struct key_pool;
+struct qnode;
+struct qnode_pool;
+struct lthread_sched;
+struct lthread_tls;
+
+
+#define BIT(x) (1 << (x))
+#define CLEARBIT(x) ~(1 << (x))
+
+#define POSIX_ERRNO(x) (x)
+
+#define MAX_LTHREAD_NAME_SIZE 64
+
+#define RTE_LOGTYPE_LTHREAD RTE_LOGTYPE_USER1
+
+
+/* define some shorthand for current scheduler and current thread */
+#define THIS_SCHED RTE_PER_LCORE(this_sched)
+#define THIS_LTHREAD RTE_PER_LCORE(this_sched)->current_lthread
+
+/*
+ * Definition of an scheduler struct
+ */
+struct lthread_sched {
+ struct ctx ctx; /* cpu context */
+ uint64_t birth; /* time created */
+ struct lthread *current_lthread; /* running thread */
+ unsigned lcore_id; /* this sched lcore */
+ int run_flag; /* sched shutdown */
+ uint64_t nb_blocked_threads; /* blocked threads */
+ struct lthread_queue *ready; /* local ready queue */
+ struct lthread_queue *pready; /* peer ready queue */
+ struct lthread_objcache *lthread_cache; /* free lthreads */
+ struct lthread_objcache *stack_cache; /* free stacks */
+ struct lthread_objcache *per_lthread_cache; /* free per lthread */
+ struct lthread_objcache *tls_cache; /* free TLS */
+ struct lthread_objcache *cond_cache; /* free cond vars */
+ struct lthread_objcache *mutex_cache; /* free mutexes */
+ struct qnode_pool *qnode_pool; /* pool of queue nodes */
+ struct key_pool *key_pool; /* pool of free TLS keys */
+ size_t stack_size;
+ uint64_t diag_ref; /* diag ref */
+} __rte_cache_aligned;
+
+RTE_DECLARE_PER_LCORE(struct lthread_sched *, this_sched);
+
+
+/*
+ * State for an lthread
+ */
+enum lthread_st {
+ ST_LT_INIT, /* initial state */
+ ST_LT_READY, /* lthread is ready to run */
+ ST_LT_SLEEPING, /* lthread is sleeping */
+ ST_LT_EXPIRED, /* lthread timeout has expired */
+ ST_LT_EXITED, /* lthread has exited and needs cleanup */
+ ST_LT_DETACH, /* lthread frees on exit*/
+ ST_LT_CANCELLED, /* lthread has been cancelled */
+};
+
+/*
+ * lthread sub states for exit/join
+ */
+enum join_st {
+ LT_JOIN_INITIAL, /* initial state */
+ LT_JOIN_EXITING, /* thread is exiting */
+ LT_JOIN_THREAD_SET, /* joining thread has been set */
+ LT_JOIN_EXIT_VAL_SET, /* exiting thread has set ret val */
+ LT_JOIN_EXIT_VAL_READ, /* joining thread has collected ret val */
+};
+
+/* defnition of an lthread stack object */
+struct lthread_stack {
+ uint8_t stack[LTHREAD_MAX_STACK_SIZE];
+ size_t stack_size;
+ struct lthread_sched *root_sched;
+} __rte_cache_aligned;
+
+/*
+ * Definition of an lthread
+ */
+struct lthread {
+ struct ctx ctx; /* cpu context */
+
+ uint64_t state; /* current lthread state */
+
+ struct lthread_sched *sched; /* current scheduler */
+ void *stack; /* ptr to actual stack */
+ size_t stack_size; /* current stack_size */
+ size_t last_stack_size; /* last yield stack_size */
+ lthread_func_t fun; /* func ctx is running */
+ void *arg; /* func args passed to func */
+ void *per_lthread_data; /* per lthread user data */
+ lthread_exit_func exit_handler; /* called when thread exits */
+ uint64_t birth; /* time lthread was born */
+ struct lthread_queue *pending_wr_queue; /* deferred queue to write */
+ struct lthread *lt_join; /* lthread to join on */
+ uint64_t join; /* state for joining */
+ void **lt_exit_ptr; /* exit ptr for lthread_join */
+ struct lthread_sched *root_sched; /* thread was created here*/
+ struct queue_node *qnode; /* node when in a queue */
+ struct rte_timer tim; /* sleep timer */
+ struct lthread_tls *tls; /* keys in use by the thread */
+ struct lthread_stack *stack_container; /* stack */
+ char funcname[MAX_LTHREAD_NAME_SIZE]; /* thread func name */
+ uint64_t diag_ref; /* ref to user diag data */
+} __rte_cache_aligned;
+
+/*
+ * Assert
+ */
+#if LTHREAD_DIAG
+#define LTHREAD_ASSERT(expr) do { \
+ if (!(expr)) \
+ rte_panic("line%d\tassert \"" #expr "\" failed\n", __LINE__);\
+} while (0)
+#else
+#define LTHREAD_ASSERT(expr) do {} while (0)
+#endif
+
+#endif /* LTHREAD_INT_H */
diff --git a/examples/performance-thread/common/lthread_mutex.c b/examples/performance-thread/common/lthread_mutex.c
new file mode 100644
index 0000000..b532280
--- /dev/null
+++ b/examples/performance-thread/common/lthread_mutex.c
@@ -0,0 +1,255 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <limits.h>
+#include <inttypes.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <fcntl.h>
+#include <sys/time.h>
+#include <sys/mman.h>
+
+#include <rte_config.h>
+#include <rte_per_lcore.h>
+#include <rte_log.h>
+#include <rte_spinlock.h>
+#include <rte_common.h>
+
+#include "lthread_api.h"
+#include "lthread_int.h"
+#include "lthread_mutex.h"
+#include "lthread_sched.h"
+#include "lthread_queue.h"
+#include "lthread_objcache.h"
+#include "lthread_diag.h"
+
+/*
+ * Create a mutex
+ */
+int
+lthread_mutex_init(char *name, struct lthread_mutex **mutex,
+ __rte_unused const struct lthread_mutexattr *attr)
+{
+ struct lthread_mutex *m;
+
+ if (mutex == NULL)
+ return POSIX_ERRNO(EINVAL);
+
+
+ m = _lthread_objcache_alloc((THIS_SCHED)->mutex_cache);
+ if (m == NULL)
+ return POSIX_ERRNO(EAGAIN);
+
+ m->blocked = _lthread_queue_create("blocked queue");
+ if (m->blocked == NULL) {
+ _lthread_objcache_free((THIS_SCHED)->mutex_cache, m);
+ return POSIX_ERRNO(EAGAIN);
+ }
+
+ if (name == NULL)
+ strncpy(m->name, "no name", sizeof(m->name));
+ else
+ strncpy(m->name, name, sizeof(m->name));
+ m->name[sizeof(m->name)-1] = 0;
+
+ m->root_sched = THIS_SCHED;
+ m->owner = NULL;
+
+ rte_atomic64_init(&m->count);
+
+ DIAG_CREATE_EVENT(m, LT_DIAG_MUTEX_CREATE);
+ /* success */
+ (*mutex) = m;
+ return 0;
+}
+
+/*
+ * Destroy a mutex
+ */
+int lthread_mutex_destroy(struct lthread_mutex *m)
+{
+ if ((m == NULL) || (m->blocked == NULL)) {
+ DIAG_EVENT(m, LT_DIAG_MUTEX_DESTROY, m, POSIX_ERRNO(EINVAL));
+ return POSIX_ERRNO(EINVAL);
+ }
+
+ if (m->owner == NULL) {
+ /* try to delete the blocked queue */
+ if (_lthread_queue_destroy(m->blocked) < 0) {
+ DIAG_EVENT(m, LT_DIAG_MUTEX_DESTROY,
+ m, POSIX_ERRNO(EBUSY));
+ return POSIX_ERRNO(EBUSY);
+ }
+
+ /* free the mutex to cache */
+ _lthread_objcache_free(m->root_sched->mutex_cache, m);
+ DIAG_EVENT(m, LT_DIAG_MUTEX_DESTROY, m, 0);
+ return 0;
+ }
+ /* can't do its still in use */
+ DIAG_EVENT(m, LT_DIAG_MUTEX_DESTROY, m, POSIX_ERRNO(EBUSY));
+ return POSIX_ERRNO(EBUSY);
+}
+
+/*
+ * Try to obtain a mutex
+ */
+int lthread_mutex_lock(struct lthread_mutex *m)
+{
+ struct lthread *lt = THIS_LTHREAD;
+
+ if ((m == NULL) || (m->blocked == NULL)) {
+ DIAG_EVENT(m, LT_DIAG_MUTEX_LOCK, m, POSIX_ERRNO(EINVAL));
+ return POSIX_ERRNO(EINVAL);
+ }
+
+ /* allow no recursion */
+ if (m->owner == lt) {
+ DIAG_EVENT(m, LT_DIAG_MUTEX_LOCK, m, POSIX_ERRNO(EDEADLK));
+ return POSIX_ERRNO(EDEADLK);
+ }
+
+ for (;;) {
+ rte_atomic64_inc(&m->count);
+ do {
+ if (rte_atomic64_cmpset
+ ((uint64_t *) &m->owner, 0, (uint64_t) lt)) {
+ /* happy days, we got the lock */
+ DIAG_EVENT(m, LT_DIAG_MUTEX_LOCK, m, 0);
+ return 0;
+ }
+ /* spin due to race with unlock when
+ * nothing was blocked
+ */
+ } while ((rte_atomic64_read(&m->count) == 1) &&
+ (m->owner == NULL));
+
+ /* queue the current thread in the blocked queue
+ * we defer this to after we return to the scheduler
+ * to ensure that the current thread context is saved
+ * before unlock could result in it being dequeued and
+ * resumed
+ */
+ DIAG_EVENT(m, LT_DIAG_MUTEX_BLOCKED, m, lt);
+ lt->pending_wr_queue = m->blocked;
+ /* now relinquish cpu */
+ _suspend();
+ /* resumed, must loop and compete for the lock again */
+ }
+ LTHREAD_ASSERT(0);
+ return 0;
+}
+
+/* try to lock a mutex but dont block */
+int lthread_mutex_trylock(struct lthread_mutex *m)
+{
+ struct lthread *lt = THIS_LTHREAD;
+
+ if ((m == NULL) || (m->blocked == NULL)) {
+ DIAG_EVENT(m, LT_DIAG_MUTEX_TRYLOCK, m, POSIX_ERRNO(EINVAL));
+ return POSIX_ERRNO(EINVAL);
+ }
+
+ if (m->owner == lt) {
+ /* no recursion */
+ DIAG_EVENT(m, LT_DIAG_MUTEX_TRYLOCK, m, POSIX_ERRNO(EDEADLK));
+ return POSIX_ERRNO(EDEADLK);
+ }
+
+ rte_atomic64_inc(&m->count);
+ if (rte_atomic64_cmpset
+ ((uint64_t *) &m->owner, (uint64_t) NULL, (uint64_t) lt)) {
+ /* got the lock */
+ DIAG_EVENT(m, LT_DIAG_MUTEX_TRYLOCK, m, 0);
+ return 0;
+ }
+
+ /* failed so return busy */
+ rte_atomic64_dec(&m->count);
+ DIAG_EVENT(m, LT_DIAG_MUTEX_TRYLOCK, m, POSIX_ERRNO(EBUSY));
+ return POSIX_ERRNO(EBUSY);
+}
+
+/*
+ * Unlock a mutex
+ */
+int lthread_mutex_unlock(struct lthread_mutex *m)
+{
+ struct lthread *lt = THIS_LTHREAD;
+ struct lthread *unblocked;
+
+ if ((m == NULL) || (m->blocked == NULL)) {
+ DIAG_EVENT(m, LT_DIAG_MUTEX_UNLOCKED, m, POSIX_ERRNO(EINVAL));
+ return POSIX_ERRNO(EINVAL);
+ }
+
+ /* fail if its owned */
+ if (m->owner != lt || m->owner == NULL) {
+ DIAG_EVENT(m, LT_DIAG_MUTEX_UNLOCKED, m, POSIX_ERRNO(EPERM));
+ return POSIX_ERRNO(EPERM);
+ }
+
+ rte_atomic64_dec(&m->count);
+ /* if there are blocked threads then make one ready */
+ while (rte_atomic64_read(&m->count) > 0) {
+ unblocked = _lthread_queue_remove(m->blocked);
+
+ if (unblocked != NULL) {
+ rte_atomic64_dec(&m->count);
+ DIAG_EVENT(m, LT_DIAG_MUTEX_UNLOCKED, m, unblocked);
+ LTHREAD_ASSERT(unblocked->sched != NULL);
+ _ready_queue_insert((struct lthread_sched *)
+ unblocked->sched, unblocked);
+ break;
+ }
+ }
+ /* release the lock */
+ m->owner = NULL;
+ return 0;
+}
+
+/*
+ * return the diagnostic ref val stored in a mutex
+ */
+uint64_t
+lthread_mutex_diag_ref(struct lthread_mutex *m)
+{
+ if (m == NULL)
+ return 0;
+ return m->diag_ref;
+}
diff --git a/examples/performance-thread/common/lthread_mutex.h b/examples/performance-thread/common/lthread_mutex.h
new file mode 100644
index 0000000..4d30b2e
--- /dev/null
+++ b/examples/performance-thread/common/lthread_mutex.h
@@ -0,0 +1,52 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#ifndef LTHREAD_MUTEX_H_
+#define LTHREAD_MUTEX_H_
+
+#include "lthread_queue.h"
+
+
+#define MAX_MUTEX_NAME_SIZE 64
+
+struct lthread_mutex {
+ struct lthread *owner;
+ rte_atomic64_t count;
+ struct lthread_queue *blocked __rte_cache_aligned;
+ struct lthread_sched *root_sched;
+ char name[MAX_MUTEX_NAME_SIZE];
+ uint64_t diag_ref; /* optional ref to user diag data */
+} __rte_cache_aligned;
+
+#endif /* LTHREAD_MUTEX_H_ */
diff --git a/examples/performance-thread/common/lthread_objcache.h b/examples/performance-thread/common/lthread_objcache.h
new file mode 100644
index 0000000..37bbf8a
--- /dev/null
+++ b/examples/performance-thread/common/lthread_objcache.h
@@ -0,0 +1,160 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef LTHREAD_OBJCACHE_H_
+#define LTHREAD_OBJCACHE_H_
+
+#include <string.h>
+
+#include <rte_per_lcore.h>
+#include <rte_malloc.h>
+#include <rte_memory.h>
+
+#include "lthread_int.h"
+#include "lthread_diag.h"
+#include "lthread_queue.h"
+
+
+#define DISPLAY_OBCACHE_QUEUES 0
+
+RTE_DECLARE_PER_LCORE(struct lthread_sched *, this_sched);
+
+struct lthread_objcache {
+ struct lthread_queue *q;
+ size_t obj_size;
+ int prealloc_size;
+ char name[LT_MAX_NAME_SIZE];
+
+ DIAG_COUNT_DEFINE(rd);
+ DIAG_COUNT_DEFINE(wr);
+ DIAG_COUNT_DEFINE(prealloc);
+ DIAG_COUNT_DEFINE(capacity);
+ DIAG_COUNT_DEFINE(available);
+};
+
+/*
+ * Create a cache
+ */
+static inline struct
+lthread_objcache *_lthread_objcache_create(const char *name,
+ size_t obj_size,
+ int prealloc_size)
+{
+ struct lthread_objcache *c =
+ rte_malloc_socket(NULL, sizeof(struct lthread_objcache),
+ RTE_CACHE_LINE_SIZE,
+ rte_socket_id());
+ if (c == NULL)
+ return NULL;
+
+ c->q = _lthread_queue_create("cache queue");
+ if (c->q == NULL) {
+ rte_free(c);
+ return NULL;
+ }
+ c->obj_size = obj_size;
+ c->prealloc_size = prealloc_size;
+
+ if (name != NULL)
+ strncpy(c->name, name, LT_MAX_NAME_SIZE);
+ c->name[sizeof(c->name)-1] = 0;
+
+ DIAG_COUNT_INIT(c, rd);
+ DIAG_COUNT_INIT(c, wr);
+ DIAG_COUNT_INIT(c, prealloc);
+ DIAG_COUNT_INIT(c, capacity);
+ DIAG_COUNT_INIT(c, available);
+ return c;
+}
+
+/*
+ * Destroy an objcache
+ */
+static inline int
+_lthread_objcache_destroy(struct lthread_objcache *c)
+{
+ if (_lthread_queue_destroy(c->q) == 0) {
+ rte_free(c);
+ return 0;
+ }
+ return -1;
+}
+
+/*
+ * Allocate an object from an object cache
+ */
+static inline void *
+_lthread_objcache_alloc(struct lthread_objcache *c)
+{
+ int i;
+ void *data;
+ struct lthread_queue *q = c->q;
+ size_t obj_size = c->obj_size;
+ int prealloc_size = c->prealloc_size;
+
+ data = _lthread_queue_remove(q);
+
+ if (data == NULL) {
+ DIAG_COUNT_INC(c, prealloc);
+ for (i = 0; i < prealloc_size; i++) {
+ data =
+ rte_zmalloc_socket(NULL, obj_size,
+ RTE_CACHE_LINE_SIZE,
+ rte_socket_id());
+ if (data == NULL)
+ return NULL;
+
+ DIAG_COUNT_INC(c, available);
+ DIAG_COUNT_INC(c, capacity);
+ _lthread_queue_insert_mp(q, data);
+ }
+ data = _lthread_queue_remove(q);
+ }
+ DIAG_COUNT_INC(c, rd);
+ DIAG_COUNT_DEC(c, available);
+ return data;
+}
+
+/*
+ * free an object to a cache
+ */
+static inline void
+_lthread_objcache_free(struct lthread_objcache *c, void *obj)
+{
+ DIAG_COUNT_INC(c, wr);
+ DIAG_COUNT_INC(c, available);
+ _lthread_queue_insert_mp(c->q, obj);
+}
+
+
+
+#endif /* LTHREAD_OBJCACHE_H_ */
diff --git a/examples/performance-thread/common/lthread_pool.h b/examples/performance-thread/common/lthread_pool.h
new file mode 100644
index 0000000..67bfb62
--- /dev/null
+++ b/examples/performance-thread/common/lthread_pool.h
@@ -0,0 +1,333 @@
+/*
+ *-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Some portions of this software is derived from the producer
+ * consumer queues described by Dmitry Vyukov and published here
+ * http://www.1024cores.net
+ *
+ * Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DMITRY VYUKOV OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+ * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and documentation are
+ * those of the authors and should not be interpreted as representing official
+ * policies, either expressed or implied, of Dmitry Vyukov.
+ */
+
+#ifndef LTHREAD_POOL_H_
+#define LTHREAD_POOL_H_
+
+#include <rte_malloc.h>
+#include <rte_per_lcore.h>
+#include <rte_log.h>
+
+#include "lthread_int.h"
+#include "lthread_diag.h"
+#include "atomic.h"
+
+/*
+ * This file implements pool of queue nodes used by the queue implemented
+ * in lthread_queue.h.
+ *
+ * The pool is an intrusive lock free MPSC queue.
+ *
+ * The pool is created empty and populated lazily, i.e. on first attempt to
+ * allocate a the pool.
+ *
+ * Whenever the pool is empty more nodes are added to the pool
+ * The number of nodes preallocated in this way is a parameter of
+ * _qnode_pool_create. Freeing an object returns it to the pool.
+ *
+ * Each lthread scheduler maintains its own pool of nodes. L-threads must always
+ * allocate from this local pool ( because it is a single consumer queue ).
+ * L-threads can free nodes to any pool (because it is a multi producer queue)
+ * This enables threads that have affined to a different scheduler to free
+ * nodes safely.
+ */
+
+struct qnode;
+struct qnode_cache;
+
+/*
+ * define intermediate node
+ */
+struct qnode {
+ struct qnode *next;
+ void *data;
+ struct qnode_pool *pool;
+} __rte_cache_aligned;
+
+/*
+ * a pool structure
+ */
+struct qnode_pool {
+ struct qnode *head;
+ struct qnode *stub;
+ struct qnode *fast_alloc;
+ struct qnode *tail __rte_cache_aligned;
+ int pre_alloc;
+ char name[LT_MAX_NAME_SIZE];
+
+ DIAG_COUNT_DEFINE(rd);
+ DIAG_COUNT_DEFINE(wr);
+ DIAG_COUNT_DEFINE(available);
+ DIAG_COUNT_DEFINE(prealloc);
+ DIAG_COUNT_DEFINE(capacity);
+} __rte_cache_aligned;
+
+/*
+ * Create a pool of qnodes
+ */
+
+static inline struct qnode_pool *
+_qnode_pool_create(const char *name, int prealloc_size) {
+
+ struct qnode_pool *p = rte_malloc_socket(NULL,
+ sizeof(struct qnode_pool),
+ RTE_CACHE_LINE_SIZE,
+ rte_socket_id());
+
+ LTHREAD_ASSERT(p);
+
+ p->stub = rte_malloc_socket(NULL,
+ sizeof(struct qnode),
+ RTE_CACHE_LINE_SIZE,
+ rte_socket_id());
+
+ LTHREAD_ASSERT(p->stub);
+
+ if (name != NULL)
+ strncpy(p->name, name, LT_MAX_NAME_SIZE);
+ p->name[sizeof(p->name)-1] = 0;
+
+ p->stub->pool = p;
+ p->stub->next = NULL;
+ p->tail = p->stub;
+ p->head = p->stub;
+ p->pre_alloc = prealloc_size;
+
+ DIAG_COUNT_INIT(p, rd);
+ DIAG_COUNT_INIT(p, wr);
+ DIAG_COUNT_INIT(p, available);
+ DIAG_COUNT_INIT(p, prealloc);
+ DIAG_COUNT_INIT(p, capacity);
+
+ return p;
+}
+
+
+/*
+ * Insert a node into the pool
+ */
+static inline void __attribute__ ((always_inline))
+_qnode_pool_insert(struct qnode_pool *p, struct qnode *n)
+{
+ n->next = NULL;
+ struct qnode *prev = n;
+ /* We insert at the head */
+ prev = (struct qnode *) atomic64_xchg((uint64_t *)&p->head,
+ (uint64_t) prev);
+ /* there is a window of inconsistency until prev next is set */
+ /* which is why remove must retry */
+ prev->next = (n);
+}
+
+/*
+ * Remove a node from the pool
+ *
+ * There is a race with _qnode_pool_insert() whereby the queue could appear
+ * empty during a concurrent insert, this is handled by retrying
+ *
+ * The queue uses a stub node, which must be swung as the queue becomes
+ * empty, this requires an insert of the stub, which means that removing the
+ * last item from the queue incurs the penalty of an atomic exchange. Since the
+ * pool is maintained with a bulk pre-allocation the cost of this is amortised.
+ */
+static inline struct qnode *__attribute__ ((always_inline))
+_pool_remove(struct qnode_pool *p)
+{
+ struct qnode *head;
+ struct qnode *tail = p->tail;
+ struct qnode *next = tail->next;
+
+ /* we remove from the tail */
+ if (tail == p->stub) {
+ if (next == NULL)
+ return NULL;
+ /* advance the tail */
+ p->tail = next;
+ tail = next;
+ next = next->next;
+ }
+ if (likely(next != NULL)) {
+ p->tail = next;
+ return tail;
+ }
+
+ head = p->head;
+ if (tail == head)
+ return NULL;
+
+ /* swing stub node */
+ _qnode_pool_insert(p, p->stub);
+
+ next = tail->next;
+ if (next) {
+ p->tail = next;
+ return tail;
+ }
+ return NULL;
+}
+
+
+/*
+ * This adds a retry to the _pool_remove function
+ * defined above
+ */
+static inline struct qnode *__attribute__ ((always_inline))
+_qnode_pool_remove(struct qnode_pool *p)
+{
+ struct qnode *n;
+
+ do {
+ n = _pool_remove(p);
+ if (likely(n != NULL))
+ return n;
+
+ rte_compiler_barrier();
+ } while ((p->head != p->tail) &&
+ (p->tail != p->stub));
+ return NULL;
+}
+
+/*
+ * Allocate a node from the pool
+ * If the pool is empty add mode nodes
+ */
+static inline struct qnode *__attribute__ ((always_inline))
+_qnode_alloc(void)
+{
+ struct qnode_pool *p = (THIS_SCHED)->qnode_pool;
+ int prealloc_size = p->pre_alloc;
+ struct qnode *n;
+ int i;
+
+ if (likely(p->fast_alloc != NULL)) {
+ n = p->fast_alloc;
+ p->fast_alloc = NULL;
+ return n;
+ }
+
+ n = _qnode_pool_remove(p);
+
+ if (unlikely(n == NULL)) {
+ DIAG_COUNT_INC(p, prealloc);
+ for (i = 0; i < prealloc_size; i++) {
+ n = rte_malloc_socket(NULL,
+ sizeof(struct qnode),
+ RTE_CACHE_LINE_SIZE,
+ rte_socket_id());
+ if (n == NULL)
+ return NULL;
+
+ DIAG_COUNT_INC(p, available);
+ DIAG_COUNT_INC(p, capacity);
+
+ n->pool = p;
+ _qnode_pool_insert(p, n);
+ }
+ n = _qnode_pool_remove(p);
+ }
+ n->pool = p;
+ DIAG_COUNT_INC(p, rd);
+ DIAG_COUNT_DEC(p, available);
+ return n;
+}
+
+
+
+/*
+* free a queue node to the per scheduler pool from which it came
+*/
+static inline void __attribute__ ((always_inline))
+_qnode_free(struct qnode *n)
+{
+ struct qnode_pool *p = n->pool;
+
+
+ if (unlikely(p->fast_alloc != NULL) ||
+ unlikely(n->pool != (THIS_SCHED)->qnode_pool)) {
+ DIAG_COUNT_INC(p, wr);
+ DIAG_COUNT_INC(p, available);
+ _qnode_pool_insert(p, n);
+ return;
+ }
+ p->fast_alloc = n;
+}
+
+/*
+ * Destroy an qnode pool
+ * queue must be empty when this is called
+ */
+static inline int
+_qnode_pool_destroy(struct qnode_pool *p)
+{
+ rte_free(p->stub);
+ rte_free(p);
+ return 0;
+}
+
+
+#endif /* LTHREAD_POOL_H_ */
diff --git a/examples/performance-thread/common/lthread_queue.h b/examples/performance-thread/common/lthread_queue.h
new file mode 100644
index 0000000..58a39d7
--- /dev/null
+++ b/examples/performance-thread/common/lthread_queue.h
@@ -0,0 +1,303 @@
+/*
+ *-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Some portions of this software is derived from the producer
+ * consumer queues described by Dmitry Vyukov and published here
+ * http://www.1024cores.net
+ *
+ * Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DMITRY VYUKOV OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+ * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+ * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * The views and conclusions contained in the software and documentation are
+ * those of the authors and should not be interpreted as representing official
+ * policies, either expressed or implied, of Dmitry Vyukov.
+ */
+
+#ifndef LTHREAD_QUEUE_H_
+#define LTHREAD_QUEUE_H_
+
+#include <string.h>
+
+#include <rte_prefetch.h>
+#include <rte_per_lcore.h>
+
+#include "lthread_int.h"
+#include "lthread.h"
+#include "lthread_diag.h"
+#include "lthread_pool.h"
+#include "atomic.h"
+
+struct lthread_queue;
+
+/*
+ * This file implements an unbounded FIFO queue based on a lock free
+ * linked list.
+ *
+ * The queue is non-intrusive in that it uses intermediate nodes, and does
+ * not require these nodes to be inserted into the object being placed
+ * in the queue.
+ *
+ * This is slightly more efficient than the very similar queue in lthread_pool
+ * in that it does not have to swing a stub node as the queue becomes empty.
+ *
+ * The queue access functions allocate and free intermediate node
+ * transparently from/to a per scheduler pool ( see lthread_pool.h ).
+ *
+ * The queue provides both MPSC and SPSC insert methods
+ */
+
+/*
+ * define a queue of lthread nodes
+ */
+struct lthread_queue {
+ struct qnode *head;
+ struct qnode *tail __rte_cache_aligned;
+ struct lthread_queue *p;
+ char name[LT_MAX_NAME_SIZE];
+
+ DIAG_COUNT_DEFINE(rd);
+ DIAG_COUNT_DEFINE(wr);
+ DIAG_COUNT_DEFINE(size);
+
+} __rte_cache_aligned;
+
+
+
+static inline struct lthread_queue *
+_lthread_queue_create(const char *name)
+{
+ struct qnode *stub;
+ struct lthread_queue *new_queue;
+
+ new_queue = rte_malloc_socket(NULL, sizeof(struct lthread_queue),
+ RTE_CACHE_LINE_SIZE,
+ rte_socket_id());
+ if (new_queue == NULL)
+ return NULL;
+
+ /* allocated stub node */
+ stub = _qnode_alloc();
+ LTHREAD_ASSERT(stub);
+
+ if (name != NULL)
+ strncpy(new_queue->name, name, sizeof(new_queue->name));
+ new_queue->name[sizeof(new_queue->name)-1] = 0;
+
+ /* initialize queue as empty */
+ stub->next = NULL;
+ new_queue->head = stub;
+ new_queue->tail = stub;
+
+ DIAG_COUNT_INIT(new_queue, rd);
+ DIAG_COUNT_INIT(new_queue, wr);
+ DIAG_COUNT_INIT(new_queue, size);
+
+ return new_queue;
+}
+
+/**
+ * Return true if the queue is empty
+ */
+static inline int __attribute__ ((always_inline))
+_lthread_queue_empty(struct lthread_queue *q)
+{
+ return (q->tail == q->head);
+}
+
+
+
+/**
+ * Destroy a queue
+ * fail if queue is not empty
+ */
+static inline int _lthread_queue_destroy(struct lthread_queue *q)
+{
+ if (q == NULL)
+ return -1;
+
+ if (!_lthread_queue_empty(q))
+ return -1;
+
+ _qnode_free(q->head);
+ rte_free(q);
+ return 0;
+}
+
+RTE_DECLARE_PER_LCORE(struct lthread_sched *, this_sched);
+
+/*
+ * Insert a node into a queue
+ * this implementation is multi producer safe
+ */
+static inline struct qnode *__attribute__ ((always_inline))
+_lthread_queue_insert_mp(struct lthread_queue
+ *q, void *data)
+{
+ struct qnode *prev;
+ struct qnode *n = _qnode_alloc();
+
+ if (n == NULL)
+ return NULL;
+
+ /* set object in node */
+ n->data = data;
+ n->next = NULL;
+
+ /* this is an MPSC method, perform a locked update */
+ prev = n;
+ prev =
+ (struct qnode *)atomic64_xchg((uint64_t *) &(q)->head,
+ (uint64_t) prev);
+ /* there is a window of inconsistency until prev next is set,
+ * which is why remove must retry
+ */
+ prev->next = n;
+
+ DIAG_COUNT_INC(q, wr);
+ DIAG_COUNT_INC(q, size);
+
+ return n;
+}
+
+/*
+ * Insert an node into a queue in single producer mode
+ * this implementation is NOT mult producer safe
+ */
+static inline struct qnode *__attribute__ ((always_inline))
+_lthread_queue_insert_sp(struct lthread_queue
+ *q, void *data)
+{
+ /* allocate a queue node */
+ struct qnode *prev;
+ struct qnode *n = _qnode_alloc();
+
+ if (n == NULL)
+ return NULL;
+
+ /* set data in node */
+ n->data = data;
+ n->next = NULL;
+
+ /* this is an SPSC method, no need for locked exchange operation */
+ prev = q->head;
+ prev->next = q->head = n;
+
+ DIAG_COUNT_INC(q, wr);
+ DIAG_COUNT_INC(q, size);
+
+ return n;
+}
+
+/*
+ * Remove a node from a queue
+ */
+static inline void *__attribute__ ((always_inline))
+_lthread_queue_poll(struct lthread_queue *q)
+{
+ void *data = NULL;
+ struct qnode *tail = q->tail;
+ struct qnode *next = (struct qnode *)tail->next;
+ /*
+ * There is a small window of inconsistency between producer and
+ * consumer whereby the queue may appear empty if consumer and
+ * producer access it at the same time.
+ * The consumer must handle this by retrying
+ */
+
+ if (likely(next != NULL)) {
+ q->tail = next;
+ tail->data = next->data;
+ data = tail->data;
+
+ /* free the node */
+ _qnode_free(tail);
+
+ DIAG_COUNT_INC(q, rd);
+ DIAG_COUNT_DEC(q, size);
+ return data;
+ }
+ return NULL;
+}
+
+/*
+ * Remove a node from a queue
+ */
+static inline void *__attribute__ ((always_inline))
+_lthread_queue_remove(struct lthread_queue *q)
+{
+ void *data = NULL;
+
+ /*
+ * There is a small window of inconsistency between producer and
+ * consumer whereby the queue may appear empty if consumer and
+ * producer access it at the same time. We handle this by retrying
+ */
+ do {
+ data = _lthread_queue_poll(q);
+
+ if (likely(data != NULL)) {
+
+ DIAG_COUNT_INC(q, rd);
+ DIAG_COUNT_DEC(q, size);
+ return data;
+ }
+ rte_compiler_barrier();
+ } while (unlikely(!_lthread_queue_empty(q)));
+ return NULL;
+}
+
+
+#endif /* LTHREAD_QUEUE_H_ */
diff --git a/examples/performance-thread/common/lthread_sched.c b/examples/performance-thread/common/lthread_sched.c
new file mode 100644
index 0000000..3fad768
--- /dev/null
+++ b/examples/performance-thread/common/lthread_sched.c
@@ -0,0 +1,600 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Some portions of this software is derived from the
+ * https://github.com/halayli/lthread which carrys the following license.
+ *
+ * Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+
+#define RTE_MEM 1
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <limits.h>
+#include <inttypes.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <fcntl.h>
+#include <sys/time.h>
+#include <sys/mman.h>
+#include <sched.h>
+
+#include <rte_config.h>
+#include <rte_prefetch.h>
+#include <rte_per_lcore.h>
+#include <rte_atomic.h>
+#include <rte_atomic_64.h>
+#include <rte_log.h>
+#include <rte_common.h>
+#include <rte_branch_prediction.h>
+
+#include "lthread_api.h"
+#include "lthread_int.h"
+#include "lthread_sched.h"
+#include "lthread_objcache.h"
+#include "lthread_timer.h"
+#include "lthread_mutex.h"
+#include "lthread_cond.h"
+#include "lthread_tls.h"
+#include "lthread_diag.h"
+
+/*
+ * This file implements the lthread scheduler
+ * The scheduler is the function lthread_run()
+ * This must be run as the main loop of an EAL thread.
+ *
+ * Currently once a scheduler is created it cannot be destroyed
+ * When a scheduler shuts down it is assumed that the application is terminating
+ */
+
+static rte_atomic16_t num_schedulers;
+static rte_atomic16_t active_schedulers;
+
+/* one scheduler per lcore */
+RTE_DEFINE_PER_LCORE(struct lthread_sched *, this_sched) = NULL;
+
+struct lthread_sched *schedcore[LTHREAD_MAX_LCORES];
+
+diag_callback diag_cb;
+
+uint64_t diag_mask;
+
+
+/* constructor */
+void lthread_sched_ctor(void) __attribute__ ((constructor));
+void lthread_sched_ctor(void)
+{
+ memset(schedcore, 0, sizeof(schedcore));
+ rte_atomic16_init(&num_schedulers);
+ rte_atomic16_set(&num_schedulers, 1);
+ rte_atomic16_init(&active_schedulers);
+ rte_atomic16_set(&active_schedulers, 0);
+ diag_cb = NULL;
+}
+
+
+enum sched_alloc_phase {
+ SCHED_ALLOC_OK,
+ SCHED_ALLOC_QNODE_POOL,
+ SCHED_ALLOC_READY_QUEUE,
+ SCHED_ALLOC_PREADY_QUEUE,
+ SCHED_ALLOC_LTHREAD_CACHE,
+ SCHED_ALLOC_STACK_CACHE,
+ SCHED_ALLOC_PERLT_CACHE,
+ SCHED_ALLOC_TLS_CACHE,
+ SCHED_ALLOC_COND_CACHE,
+ SCHED_ALLOC_MUTEX_CACHE,
+};
+
+static int
+_lthread_sched_alloc_resources(struct lthread_sched *new_sched)
+{
+ int alloc_status;
+
+ do {
+ /* Initialize per scheduler queue node pool */
+ alloc_status = SCHED_ALLOC_QNODE_POOL;
+ new_sched->qnode_pool =
+ _qnode_pool_create("qnode pool", LTHREAD_PREALLOC);
+ if (new_sched->qnode_pool == NULL)
+ break;
+
+ /* Initialize per scheduler local ready queue */
+ alloc_status = SCHED_ALLOC_READY_QUEUE;
+ new_sched->ready = _lthread_queue_create("ready queue");
+ if (new_sched->ready == NULL)
+ break;
+
+ /* Initialize per scheduler local peer ready queue */
+ alloc_status = SCHED_ALLOC_PREADY_QUEUE;
+ new_sched->pready = _lthread_queue_create("pready queue");
+ if (new_sched->pready == NULL)
+ break;
+
+ /* Initialize per scheduler local free lthread cache */
+ alloc_status = SCHED_ALLOC_LTHREAD_CACHE;
+ new_sched->lthread_cache =
+ _lthread_objcache_create("lthread cache",
+ sizeof(struct lthread),
+ LTHREAD_PREALLOC);
+ if (new_sched->lthread_cache == NULL)
+ break;
+
+ /* Initialize per scheduler local free stack cache */
+ alloc_status = SCHED_ALLOC_STACK_CACHE;
+ new_sched->stack_cache =
+ _lthread_objcache_create("stack_cache",
+ sizeof(struct lthread_stack),
+ LTHREAD_PREALLOC);
+ if (new_sched->stack_cache == NULL)
+ break;
+
+ /* Initialize per scheduler local free per lthread data cache */
+ alloc_status = SCHED_ALLOC_PERLT_CACHE;
+ new_sched->per_lthread_cache =
+ _lthread_objcache_create("per_lt cache",
+ RTE_PER_LTHREAD_SECTION_SIZE,
+ LTHREAD_PREALLOC);
+ if (new_sched->per_lthread_cache == NULL)
+ break;
+
+ /* Initialize per scheduler local free tls cache */
+ alloc_status = SCHED_ALLOC_TLS_CACHE;
+ new_sched->tls_cache =
+ _lthread_objcache_create("TLS cache",
+ sizeof(struct lthread_tls),
+ LTHREAD_PREALLOC);
+ if (new_sched->tls_cache == NULL)
+ break;
+
+ /* Initialize per scheduler local free cond var cache */
+ alloc_status = SCHED_ALLOC_COND_CACHE;
+ new_sched->cond_cache =
+ _lthread_objcache_create("cond cache",
+ sizeof(struct lthread_cond),
+ LTHREAD_PREALLOC);
+ if (new_sched->cond_cache == NULL)
+ break;
+
+ /* Initialize per scheduler local free mutex cache */
+ alloc_status = SCHED_ALLOC_MUTEX_CACHE;
+ new_sched->mutex_cache =
+ _lthread_objcache_create("mutex cache",
+ sizeof(struct lthread_mutex),
+ LTHREAD_PREALLOC);
+ if (new_sched->mutex_cache == NULL)
+ break;
+
+ alloc_status = SCHED_ALLOC_OK;
+ } while (0);
+
+ /* roll back on any failure */
+ switch (alloc_status) {
+ case SCHED_ALLOC_MUTEX_CACHE:
+ _lthread_objcache_destroy(new_sched->cond_cache);
+ /* fall through */
+ case SCHED_ALLOC_COND_CACHE:
+ _lthread_objcache_destroy(new_sched->tls_cache);
+ /* fall through */
+ case SCHED_ALLOC_TLS_CACHE:
+ _lthread_objcache_destroy(new_sched->per_lthread_cache);
+ /* fall through */
+ case SCHED_ALLOC_PERLT_CACHE:
+ _lthread_objcache_destroy(new_sched->stack_cache);
+ /* fall through */
+ case SCHED_ALLOC_STACK_CACHE:
+ _lthread_objcache_destroy(new_sched->lthread_cache);
+ /* fall through */
+ case SCHED_ALLOC_LTHREAD_CACHE:
+ _lthread_queue_destroy(new_sched->pready);
+ /* fall through */
+ case SCHED_ALLOC_PREADY_QUEUE:
+ _lthread_queue_destroy(new_sched->ready);
+ /* fall through */
+ case SCHED_ALLOC_READY_QUEUE:
+ _qnode_pool_destroy(new_sched->qnode_pool);
+ /* fall through */
+ case SCHED_ALLOC_QNODE_POOL:
+ /* fall through */
+ case SCHED_ALLOC_OK:
+ break;
+ }
+ return alloc_status;
+}
+
+
+/*
+ * Create a scheduler on the current lcore
+ */
+struct lthread_sched *_lthread_sched_create(size_t stack_size)
+{
+ int status;
+ struct lthread_sched *new_sched;
+ unsigned lcoreid = rte_lcore_id();
+
+ LTHREAD_ASSERT(stack_size <= LTHREAD_MAX_STACK_SIZE);
+
+ if (stack_size == 0)
+ stack_size = LTHREAD_MAX_STACK_SIZE;
+
+ new_sched =
+ rte_calloc_socket(NULL, 1, sizeof(struct lthread_sched),
+ RTE_CACHE_LINE_SIZE,
+ rte_socket_id());
+ if (new_sched == NULL) {
+ RTE_LOG(CRIT, LTHREAD,
+ "Failed to allocate memory for scheduler\n");
+ return NULL;
+ }
+
+ _lthread_key_pool_init();
+
+ new_sched->stack_size = stack_size;
+ new_sched->birth = rte_rdtsc();
+ THIS_SCHED = new_sched;
+
+ status = _lthread_sched_alloc_resources(new_sched);
+ if (status != SCHED_ALLOC_OK) {
+ RTE_LOG(CRIT, LTHREAD,
+ "Failed to allocate resources for scheduler code = %d\n",
+ status);
+ rte_free(new_sched);
+ return NULL;
+ }
+
+ bzero(&new_sched->ctx, sizeof(struct ctx));
+
+ new_sched->lcore_id = lcoreid;
+
+ schedcore[lcoreid] = new_sched;
+
+ new_sched->run_flag = 1;
+
+ DIAG_EVENT(new_sched, LT_DIAG_SCHED_CREATE, rte_lcore_id(), 0);
+
+ rte_wmb();
+ return new_sched;
+}
+
+/*
+ * Set the number of schedulers in the system
+ */
+int lthread_num_schedulers_set(int num)
+{
+ rte_atomic16_set(&num_schedulers, num);
+ return (int)rte_atomic16_read(&num_schedulers);
+}
+
+/*
+ * Return the number of schedulers active
+ */
+int lthread_active_schedulers(void)
+{
+ return (int)rte_atomic16_read(&active_schedulers);
+}
+
+
+/**
+ * shutdown the scheduler running on the specified lcore
+ */
+void lthread_scheduler_shutdown(unsigned lcoreid)
+{
+ uint64_t coreid = (uint64_t) lcoreid;
+
+ if (coreid < LTHREAD_MAX_LCORES) {
+ if (schedcore[coreid] != NULL)
+ schedcore[coreid]->run_flag = 0;
+ }
+}
+
+/**
+ * shutdown all schedulers
+ */
+void lthread_scheduler_shutdown_all(void)
+{
+ uint64_t i;
+
+ /*
+ * give time for all schedulers to have started
+ * Note we use sched_yield() rather than pthread_yield() to allow
+ * for the possibility of a pthread wrapper on lthread_yield(),
+ * something that is not possible unless the scheduler is running.
+ */
+ while (rte_atomic16_read(&active_schedulers) <
+ rte_atomic16_read(&num_schedulers))
+ sched_yield();
+
+ for (i = 0; i < LTHREAD_MAX_LCORES; i++) {
+ if (schedcore[i] != NULL)
+ schedcore[i]->run_flag = 0;
+ }
+}
+
+/*
+ * Resume a suspended lthread
+ */
+static inline void
+_lthread_resume(struct lthread *lt) __attribute__ ((always_inline));
+static inline void _lthread_resume(struct lthread *lt)
+{
+ struct lthread_sched *sched = THIS_SCHED;
+ struct lthread_stack *s;
+ uint64_t state = lt->state;
+#if LTHREAD_DIAG
+ int init = 0;
+#endif
+
+ sched->current_lthread = lt;
+
+ if (state & (BIT(ST_LT_CANCELLED) | BIT(ST_LT_EXITED))) {
+ /* if detached we can free the thread now */
+ if (state & BIT(ST_LT_DETACH)) {
+ _lthread_free(lt);
+ sched->current_lthread = NULL;
+ return;
+ }
+ }
+
+ if (state & BIT(ST_LT_INIT)) {
+ /* first time this thread has been run */
+ /* assign thread to this scheduler */
+ lt->sched = THIS_SCHED;
+
+ /* allocate stack */
+ s = _stack_alloc();
+
+ lt->stack_container = s;
+ _lthread_set_stack(lt, s->stack, s->stack_size);
+
+ /* allocate memory for TLS used by this thread */
+ _lthread_tls_alloc(lt);
+
+ lt->state = BIT(ST_LT_READY);
+#if LTHREAD_DIAG
+ init = 1;
+#endif
+ }
+
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_RESUMED, init, lt);
+
+ /* switch to the new thread */
+ ctx_switch(<->ctx, &sched->ctx);
+
+ /* If posting to a queue that could be read by another lcore
+ * we defer the queue write till now to ensure the context has been
+ * saved before the other core tries to resume it
+ * This applies to blocking on mutex, cond, and to set_affinity
+ */
+ if (lt->pending_wr_queue != NULL) {
+ struct lthread_queue *dest = lt->pending_wr_queue;
+
+ lt->pending_wr_queue = NULL;
+
+ /* queue the current thread to the specified queue */
+ _lthread_queue_insert_mp(dest, lt);
+ }
+
+ sched->current_lthread = NULL;
+}
+
+/*
+ * Handle sleep timer expiry
+*/
+void
+_sched_timer_cb(struct rte_timer *tim, void *arg)
+{
+ struct lthread *lt = (struct lthread *) arg;
+ uint64_t state = lt->state;
+
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_TMR_EXPIRED, <->tim, 0);
+
+ rte_timer_stop(tim);
+
+ if (lt->state & BIT(ST_LT_CANCELLED))
+ (THIS_SCHED)->nb_blocked_threads--;
+
+ lt->state = state | BIT(ST_LT_EXPIRED);
+ _lthread_resume(lt);
+ lt->state = state & CLEARBIT(ST_LT_EXPIRED);
+}
+
+
+
+/*
+ * Returns 0 if there is a pending job in scheduler or 1 if done and can exit.
+ */
+static inline int _lthread_sched_isdone(struct lthread_sched *sched)
+{
+ return ((sched->run_flag == 0) &&
+ (_lthread_queue_empty(sched->ready)) &&
+ (_lthread_queue_empty(sched->pready)) &&
+ (sched->nb_blocked_threads == 0));
+}
+
+/*
+ * Wait for all schedulers to start
+ */
+static inline void _lthread_schedulers_sync_start(void)
+{
+ rte_atomic16_inc(&active_schedulers);
+
+ /* wait for lthread schedulers
+ * Note we use sched_yield() rather than pthread_yield() to allow
+ * for the possibility of a pthread wrapper on lthread_yield(),
+ * something that is not possible unless the scheduler is running.
+ */
+ while (rte_atomic16_read(&active_schedulers) <
+ rte_atomic16_read(&num_schedulers))
+ sched_yield();
+
+}
+
+/*
+ * Wait for all schedulers to stop
+ */
+static inline void _lthread_schedulers_sync_stop(void)
+{
+ rte_atomic16_dec(&active_schedulers);
+ rte_atomic16_dec(&num_schedulers);
+
+ /* wait for schedulers
+ * Note we use sched_yield() rather than pthread_yield() to allow
+ * for the possibility of a pthread wrapper on lthread_yield(),
+ * something that is not possible unless the scheduler is running.
+ */
+ while (rte_atomic16_read(&active_schedulers) > 0)
+ sched_yield();
+
+}
+
+
+/*
+ * Run the lthread scheduler
+ * This loop is the heart of the system
+ */
+void lthread_run(void)
+{
+
+ struct lthread_sched *sched = THIS_SCHED;
+ struct lthread *lt = NULL;
+
+ RTE_LOG(INFO, LTHREAD,
+ "starting scheduler %p on lcore %u phys core %u\n",
+ sched, rte_lcore_id(),
+ rte_lcore_index(rte_lcore_id()));
+
+ /* if more than one, wait for all schedulers to start */
+ _lthread_schedulers_sync_start();
+
+
+ /*
+ * This is the main scheduling loop
+ * So long as there are tasks in existence we run this loop.
+ * We check for:-
+ * expired timers,
+ * the local ready queue,
+ * and the peer ready queue,
+ *
+ * and resume lthreads ad infinitum.
+ */
+ while (!_lthread_sched_isdone(sched)) {
+
+ rte_timer_manage();
+
+ lt = _lthread_queue_poll(sched->ready);
+ if (lt != NULL)
+ _lthread_resume(lt);
+ lt = _lthread_queue_poll(sched->pready);
+ if (lt != NULL)
+ _lthread_resume(lt);
+ }
+
+
+ /* if more than one wait for all schedulers to stop */
+ _lthread_schedulers_sync_stop();
+
+ (THIS_SCHED) = NULL;
+
+ RTE_LOG(INFO, LTHREAD,
+ "stopping scheduler %p on lcore %u phys core %u\n",
+ sched, rte_lcore_id(),
+ rte_lcore_index(rte_lcore_id()));
+ fflush(stdout);
+}
+
+/*
+ * Return the scheduler for this lcore
+ *
+ */
+struct lthread_sched *_lthread_sched_get(int lcore_id)
+{
+ if (lcore_id > LTHREAD_MAX_LCORES)
+ return NULL;
+ return schedcore[lcore_id];
+}
+
+/*
+ * migrate the current thread to another scheduler running
+ * on the specified lcore.
+ */
+int lthread_set_affinity(unsigned lcoreid)
+{
+ struct lthread *lt = THIS_LTHREAD;
+ struct lthread_sched *dest_sched;
+
+ if (unlikely(lcoreid > LTHREAD_MAX_LCORES))
+ return POSIX_ERRNO(EINVAL);
+
+
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_AFFINITY, lcoreid, 0);
+
+ dest_sched = schedcore[lcoreid];
+
+ if (unlikely(dest_sched == NULL))
+ return POSIX_ERRNO(EINVAL);
+
+ if (likely(dest_sched != THIS_SCHED)) {
+ lt->sched = dest_sched;
+ lt->pending_wr_queue = dest_sched->pready;
+ _affinitize();
+ return 0;
+ }
+ return 0;
+}
diff --git a/examples/performance-thread/common/lthread_sched.h b/examples/performance-thread/common/lthread_sched.h
new file mode 100644
index 0000000..4ce56c2
--- /dev/null
+++ b/examples/performance-thread/common/lthread_sched.h
@@ -0,0 +1,152 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Some portions of this software is derived from the
+ * https://github.com/halayli/lthread which carrys the following license.
+ *
+ * Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef LTHREAD_SCHED_H_
+#define LTHREAD_SCHED_H_
+
+#include "lthread_int.h"
+#include "lthread_queue.h"
+#include "lthread_objcache.h"
+#include "lthread_diag.h"
+#include "ctx.h"
+
+/*
+ * insert an lthread into a queue
+ */
+static inline void
+_ready_queue_insert(struct lthread_sched *sched, struct lthread *lt)
+{
+ if (sched == THIS_SCHED)
+ _lthread_queue_insert_sp((THIS_SCHED)->ready, lt);
+ else
+ _lthread_queue_insert_mp(sched->pready, lt);
+}
+
+/*
+ * remove an lthread from a queue
+ */
+static inline struct lthread *_ready_queue_remove(struct lthread_queue *q)
+{
+ return _lthread_queue_remove(q);
+}
+
+/**
+ * Return true if the ready queue is empty
+ */
+static inline int _ready_queue_empty(struct lthread_queue *q)
+{
+ return _lthread_queue_empty(q);
+}
+
+static inline uint64_t _sched_now(void)
+{
+ uint64_t now = rte_rdtsc();
+
+ if (now > (THIS_SCHED)->birth)
+ return now - (THIS_SCHED)->birth;
+ if (now < (THIS_SCHED)->birth)
+ return (THIS_SCHED)->birth - now;
+ /* never return 0 because this means sleep forever */
+ return 1;
+}
+
+static inline void
+_affinitize(void) __attribute__ ((always_inline));
+static inline void
+_affinitize(void)
+{
+ struct lthread *lt = THIS_LTHREAD;
+
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_SUSPENDED, 0, 0);
+ ctx_switch(&(THIS_SCHED)->ctx, <->ctx);
+}
+
+static inline void
+_suspend(void) __attribute__ ((always_inline));
+static inline void
+_suspend(void)
+{
+ struct lthread *lt = THIS_LTHREAD;
+
+ (THIS_SCHED)->nb_blocked_threads++;
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_SUSPENDED, 0, 0);
+ ctx_switch(&(THIS_SCHED)->ctx, <->ctx);
+ (THIS_SCHED)->nb_blocked_threads--;
+}
+
+static inline void
+_reschedule(void) __attribute__ ((always_inline));
+static inline void
+_reschedule(void)
+{
+ struct lthread *lt = THIS_LTHREAD;
+
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_RESCHEDULED, 0, 0);
+ _ready_queue_insert(THIS_SCHED, lt);
+ ctx_switch(&(THIS_SCHED)->ctx, <->ctx);
+}
+
+extern struct lthread_sched *schedcore[];
+void _sched_timer_cb(struct rte_timer *tim, void *arg);
+void _sched_shutdown(__rte_unused void *arg);
+
+
+#endif /* LTHREAD_SCHED_H_ */
diff --git a/examples/performance-thread/common/lthread_timer.h b/examples/performance-thread/common/lthread_timer.h
new file mode 100644
index 0000000..3cf82ab
--- /dev/null
+++ b/examples/performance-thread/common/lthread_timer.h
@@ -0,0 +1,78 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef LTHREAD_TIMER_H_
+#define LTHREAD_TIMER_H_
+
+#include "lthread_int.h"
+#include "lthread_sched.h"
+
+
+static inline uint64_t
+_ns_to_clks(uint64_t ns)
+{
+ unsigned __int128 clkns = rte_get_tsc_hz();
+
+ clkns *= ns;
+ clkns /= 1000000000;
+ return (uint64_t) clkns;
+}
+
+
+static inline void
+_timer_start(struct lthread *lt, uint64_t clks)
+{
+ if (clks > 0) {
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_TMR_START, <->tim, clks);
+ rte_timer_init(<->tim);
+ rte_timer_reset(<->tim,
+ clks,
+ SINGLE,
+ rte_lcore_id(),
+ _sched_timer_cb,
+ (void *)lt);
+ }
+}
+
+
+static inline void
+_timer_stop(struct lthread *lt)
+{
+ if (lt != NULL) {
+ DIAG_EVENT(lt, LT_DIAG_LTHREAD_TMR_DELETE, <->tim, 0);
+ rte_timer_stop(<->tim);
+ }
+}
+
+
+#endif /* LTHREAD_TIMER_H_ */
diff --git a/examples/performance-thread/common/lthread_tls.c b/examples/performance-thread/common/lthread_tls.c
new file mode 100644
index 0000000..ba468b8
--- /dev/null
+++ b/examples/performance-thread/common/lthread_tls.c
@@ -0,0 +1,254 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <limits.h>
+#include <inttypes.h>
+#include <unistd.h>
+#include <pthread.h>
+#include <fcntl.h>
+#include <sys/time.h>
+#include <sys/mman.h>
+#include <execinfo.h>
+#include <sched.h>
+
+#include <rte_config.h>
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_ring.h>
+#include <rte_atomic_64.h>
+
+#include "lthread_tls.h"
+#include "lthread_queue.h"
+#include "lthread_objcache.h"
+#include "lthread_sched.h"
+
+static struct rte_ring *key_pool;
+static uint64_t key_pool_init;
+
+/* needed to cause section start and end to be defined */
+RTE_DEFINE_PER_LTHREAD(void *, dummy);
+
+static struct lthread_key key_table[LTHREAD_MAX_KEYS];
+
+void lthread_tls_ctor(void) __attribute__((constructor));
+
+void lthread_tls_ctor(void)
+{
+ key_pool = NULL;
+ key_pool_init = 0;
+}
+
+/*
+ * Initialize a pool of keys
+ * These are unique tokens that can be obtained by threads
+ * calling lthread_key_create()
+ */
+void _lthread_key_pool_init(void)
+{
+ static struct rte_ring *pool;
+ struct lthread_key *new_key;
+ char name[MAX_LTHREAD_NAME_SIZE];
+
+ bzero(key_table, sizeof(key_table));
+
+ /* only one lcore should do this */
+ if (rte_atomic64_cmpset(&key_pool_init, 0, 1)) {
+
+ snprintf(name,
+ MAX_LTHREAD_NAME_SIZE,
+ "lthread_key_pool_%d",
+ getpid());
+
+ pool = rte_ring_create(name,
+ LTHREAD_MAX_KEYS, 0, 0);
+ LTHREAD_ASSERT(pool);
+
+ int i;
+
+ for (i = 1; i < LTHREAD_MAX_KEYS; i++) {
+ new_key = &key_table[i];
+ rte_ring_mp_enqueue((struct rte_ring *)pool,
+ (void *)new_key);
+ }
+ key_pool = pool;
+ }
+ /* other lcores wait here till done */
+ while (key_pool == NULL) {
+ rte_compiler_barrier();
+ sched_yield();
+ };
+}
+
+/*
+ * Create a key
+ * this means getting a key from the the pool
+ */
+int lthread_key_create(unsigned int *key, tls_destructor_func destructor)
+{
+ if (key == NULL)
+ return POSIX_ERRNO(EINVAL);
+
+ struct lthread_key *new_key;
+
+ if (rte_ring_mc_dequeue((struct rte_ring *)key_pool, (void **)&new_key)
+ == 0) {
+ new_key->destructor = destructor;
+ *key = (new_key - key_table);
+
+ return 0;
+ }
+ return POSIX_ERRNO(EAGAIN);
+}
+
+
+/*
+ * Delete a key
+ */
+int lthread_key_delete(unsigned int k)
+{
+ struct lthread_key *key;
+
+ key = (struct lthread_key *) &key_table[k];
+
+ if (k > LTHREAD_MAX_KEYS)
+ return POSIX_ERRNO(EINVAL);
+
+ key->destructor = NULL;
+ rte_ring_mp_enqueue((struct rte_ring *)key_pool,
+ (void *)key);
+ return 0;
+}
+
+
+
+/*
+ * Break association for all keys in use by this thread
+ * invoke the destructor if available.
+ * Since a destructor can create keys we could enter an infinite loop
+ * therefore we give up after LTHREAD_DESTRUCTOR_ITERATIONS
+ * the behavior is modeled on pthread
+ */
+void _lthread_tls_destroy(struct lthread *lt)
+{
+ int i, k;
+ int nb_keys;
+ void *data;
+
+ for (i = 0; i < LTHREAD_DESTRUCTOR_ITERATIONS; i++) {
+
+ for (k = 1; k < LTHREAD_MAX_KEYS; k++) {
+
+ /* no keys in use ? */
+ nb_keys = lt->tls->nb_keys_inuse;
+ if (nb_keys == 0)
+ return;
+
+ /* this key not in use ? */
+ if (lt->tls->data[k] == NULL)
+ continue;
+
+ /* remove this key */
+ data = lt->tls->data[k];
+ lt->tls->data[k] = NULL;
+ lt->tls->nb_keys_inuse = nb_keys-1;
+
+ /* invoke destructor */
+ if (key_table[k].destructor != NULL)
+ key_table[k].destructor(data);
+ }
+ }
+}
+
+/*
+ * Return the pointer associated with a key
+ * If the key is no longer valid return NULL
+ */
+void
+*lthread_getspecific(unsigned int k)
+{
+
+ if (k > LTHREAD_MAX_KEYS)
+ return NULL;
+
+ return THIS_LTHREAD->tls->data[k];
+}
+
+/*
+ * Set a value against a key
+ * If the key is no longer valid return an error
+ * when storing value
+ */
+int lthread_setspecific(unsigned int k, const void *data)
+{
+ if (k > LTHREAD_MAX_KEYS)
+ return POSIX_ERRNO(EINVAL);
+
+ int n = THIS_LTHREAD->tls->nb_keys_inuse;
+
+ /* discard const qualifier */
+ char *p = (char *) (uintptr_t) data;
+
+
+ if (data != NULL) {
+ if (THIS_LTHREAD->tls->data[k] == NULL)
+ THIS_LTHREAD->tls->nb_keys_inuse = n+1;
+ }
+
+ THIS_LTHREAD->tls->data[k] = (void *) p;
+ return 0;
+}
+
+/*
+ * Allocate data for TLS cache
+*/
+void _lthread_tls_alloc(struct lthread *lt)
+{
+ struct lthread_tls *tls;
+
+ tls = _lthread_objcache_alloc((THIS_SCHED)->tls_cache);
+
+ LTHREAD_ASSERT(tls != NULL);
+
+ tls->root_sched = (THIS_SCHED);
+ lt->tls = tls;
+
+ /* allocate data for TLS varaiables using RTE_PER_LTHREAD macros */
+ if (sizeof(void *) < (uint64_t)RTE_PER_LTHREAD_SECTION_SIZE) {
+ lt->per_lthread_data =
+ _lthread_objcache_alloc((THIS_SCHED)->per_lthread_cache);
+ }
+}
diff --git a/examples/performance-thread/common/lthread_tls.h b/examples/performance-thread/common/lthread_tls.h
new file mode 100644
index 0000000..86cbfad
--- /dev/null
+++ b/examples/performance-thread/common/lthread_tls.h
@@ -0,0 +1,57 @@
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2015 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef LTHREAD_TLS_H_
+#define LTHREAD_TLS_H_
+
+#include "lthread_api.h"
+
+#define RTE_PER_LTHREAD_SECTION_SIZE \
+(&__stop_per_lt - &__start_per_lt)
+
+struct lthread_key {
+ tls_destructor_func destructor;
+};
+
+struct lthread_tls {
+ void *data[LTHREAD_MAX_KEYS];
+ int nb_keys_inuse;
+ struct lthread_sched *root_sched;
+};
+
+void _lthread_tls_destroy(struct lthread *lt);
+void _lthread_key_pool_init(void);
+void _lthread_tls_alloc(struct lthread *lt);
+
+
+#endif /* LTHREAD_TLS_H_ */
--
2.1.4
next prev parent reply other threads:[~2015-12-03 16:21 UTC|newest]
Thread overview: 33+ messages / expand[flat|nested] mbox.gz Atom feed top
2015-12-03 16:21 [PATCH v7 0/4] examples: add performance-thread Ian Betts
2015-12-03 16:21 ` [PATCH v7 1/4] doc: add sample application guide for performance-thread Ian Betts
2015-12-03 16:21 ` Ian Betts [this message]
2015-12-04 10:34 ` [PATCH v8 0/4] examples: add performance-thread Ian Betts
2015-12-04 10:34 ` [PATCH v8 1/4] doc: add sample application guide for performance-thread Ian Betts
2015-12-07 17:58 ` [PATCH v9 0/4] examples: add performance-thread Ian Betts
2015-12-07 17:58 ` [PATCH v9 1/4] doc: add sample application guide for performance-thread Ian Betts
2015-12-07 17:58 ` [PATCH v9 2/4] examples: add lthread subsystem " Ian Betts
2015-12-07 17:58 ` [PATCH v9 3/4] examples: add l3fwd-thread example in performance-thread Ian Betts
2015-12-08 1:35 ` Thomas Monjalon
2015-12-08 1:54 ` Betts, Ian
2015-12-08 2:28 ` Thomas Monjalon
2015-12-08 1:39 ` Thomas Monjalon
2015-12-07 17:58 ` [PATCH v9 4/4] examples: add pthread_shim example to performance thread Ian Betts
2015-12-04 10:34 ` [PATCH v8 2/4] examples: add lthread subsystem forperformance-thread Ian Betts
2015-12-07 2:38 ` Thomas Monjalon
2015-12-04 10:34 ` [PATCH v8 3/4] examples: add l3fwd-thread example in performance-thread Ian Betts
2015-12-07 2:36 ` Thomas Monjalon
2015-12-07 4:46 ` Betts, Ian
2015-12-07 11:20 ` Thomas Monjalon
2015-12-04 10:34 ` [PATCH v8 4/4] examples: add pthread_shim example to performance thread Ian Betts
2015-12-04 11:21 ` [PATCH v8 0/4] examples: add performance-thread Kulasek, TomaszX
2015-12-04 18:03 ` Stephen Hemminger
2015-12-04 18:33 ` Thomas Monjalon
2015-12-05 12:06 ` Betts, Ian
2015-12-05 17:53 ` Glynn, Michael J
2015-12-05 19:47 ` Stephen Hemminger
2015-12-05 21:21 ` Thomas Monjalon
2015-12-06 6:17 ` Betts, Ian
2015-12-04 22:10 ` Betts, Ian
2015-12-03 16:21 ` [PATCH v7 3/4] examples: add l3fwd-thread example in performance-thread Ian Betts
2015-12-03 16:21 ` [PATCH v7 4/4] examples: add pthread_shim example to performance thread Ian Betts
2015-12-03 16:44 ` [PATCH v7 0/4] examples: add performance-thread Kulasek, TomaszX
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1449159683-7092-3-git-send-email-ian.betts@intel.com \
--to=ian.betts@intel.com \
--cc=dev@dpdk.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.