System hanging when using condition variables

* System hanging when using condition variables
@ 2022-09-19 21:38 Russell Johnson
  2022-09-20  6:55 ` Philippe Gerum
  2022-09-22 14:26 ` Philippe Gerum
  0 siblings, 2 replies; 26+ messages in thread
From: Russell Johnson @ 2022-09-19 21:38 UTC (permalink / raw)
  To: xenomai

[-- Attachment #1.1.1: Type: text/plain, Size: 908 bytes --]

Hello,

I have been trying to debug an issue in our app where the entire system
hangs with the following error from the kernel: "kernel:watchdog: BUG: soft
lockup - CPU#0 stuck for 22s! [kworker/0:2:594]". This happens consistently
on every run. I was able to strip down all of the relevant code into a
simple standalone app that only uses 4 pthreads, 3 EVL events, and 3 EVL
mutexes if you would like to be able to re-create the issue (I have attached
the test file). Is there anything fundamentally flawed in this logic (the
same logic worked fine previously with STL condition variables and STL
mutexes)? It appears that there becomes some kind of deadlock in the kernel
due to an EVL event and/or EVL mutex. Let me know if there is any more
information that I can provide you to help clear up the scenario. I have
spent multiple weeks tracking this issue with no luck so far.

Thanks,

Russell

[-- Attachment #1.1.2: Type: text/html, Size: 2538 bytes --]

[-- Attachment #1.2: test.cpp --]
[-- Type: application/octet-stream, Size: 6194 bytes --]

// g++ -Wall -g -o test test.cpp -I/opt/evl/include -L/opt/evl/lib -levl -lpthread

#include <evl/evl.h>
#include <evl/thread.h>
#include <evl/clock.h>
#include <thread>
#include <unistd.h>
#include <system_error>

namespace
{
    evl_event e1;
    evl_event e2;
    evl_event e3;

    struct evl_mutex m1;
    struct evl_mutex m2;
    struct evl_mutex m3;
}

void* Thread1(void*)
{
    pthread_setname_np(pthread_self(), "Thread1");

    evl_attach_thread(EVL_CLONE_OBSERVABLE | EVL_CLONE_NONBLOCK, "Thread1");

    while (true)
    {
        evl_usleep(100000);

        evl_printf("Thread 1 woken up\n");

        /* Broadcast Event 1 */

        // Lock mutex
        evl_lock_mutex(&m1);

        // Broadcast event
        evl_broadcast_event(&e1);

        // Unlock mutex
        evl_unlock_mutex(&m1);
    }
}

void* Thread2(void*)
{
    pthread_setname_np(pthread_self(), "Thread2");

    evl_attach_thread(EVL_CLONE_OBSERVABLE | EVL_CLONE_NONBLOCK, "Thread2");

    while (true)
    {
        /* Wait for event 1 */

        // Lock mutex
        evl_lock_mutex(&m1);

        // Wait
        evl_wait_event(&e1, &m1);

        // Unlock mutex
        evl_unlock_mutex(&m1);

        evl_printf("Thread 2 woken up\n");

        /* Signal Event 2 */

        // Signal 50 events
        for (size_t i = 0; i < 50; i++)
        {
            // Lock mutex
            evl_lock_mutex(&m2);

            // Signal
            evl_signal_event(&e2);

            // Unlock mutex
            evl_unlock_mutex(&m2);
        }
    }
}

void* Thread3(void*)
{
    pthread_setname_np(pthread_self(), "Thread3");

    evl_attach_thread(EVL_CLONE_OBSERVABLE | EVL_CLONE_NONBLOCK, "Thread3");

    while (true)
    {
        /* Wait for Event 2 */

        // Lock mutex
        evl_lock_mutex(&m2);

        // Wait
        evl_wait_event(&e2, &m2);

        // Unlock mutex
        evl_unlock_mutex(&m2);

        evl_printf("Thread 3 woken up\n");

        /* Broadcast Event 3 */

        // Lock mutex
        evl_lock_mutex(&m3);

        // Broadcast event
        evl_broadcast_event(&e3);

        // Unlock mutex
        evl_unlock_mutex(&m3);
    }
}

void* Thread4(void*)
{
    pthread_setname_np(pthread_self(), "Thread4");

    evl_attach_thread(EVL_CLONE_OBSERVABLE | EVL_CLONE_NONBLOCK, "Thread4");

    while (true)
    {
        /* Wait for Event 3 */

        // Lock mutex
        evl_lock_mutex(&m3);

        // Wait for 50 signals
        for (size_t i = 0; i < 50; i++)
        {
            evl_wait_event(&e3, &m3);
        }

        // Unlock mutex
        evl_unlock_mutex(&m3);

        evl_printf("Thread 4 woken up\n");
    }
}

int main(int argc, char *argv[])
{
    // Init EVL
    int ret = evl_init();
    if (ret != 0)
    {
        printf("EVL Init failed with error: %d\n", ret);
        return -1;
    }

    // Setup events
    evl_create_event(&e1, EVL_CLOCK_MONOTONIC, EVL_CLONE_PRIVATE, "Event 1");
    evl_create_event(&e2, EVL_CLOCK_MONOTONIC, EVL_CLONE_PRIVATE, "Event 2");
    evl_create_event(&e3, EVL_CLOCK_MONOTONIC, EVL_CLONE_PRIVATE, "Event 3");

    // Setup mutexes
    evl_create_mutex(&m1, EVL_CLOCK_MONOTONIC, 0, EVL_MUTEX_NORMAL|EVL_CLONE_PRIVATE, "Mutex 1");
    evl_create_mutex(&m2, EVL_CLOCK_MONOTONIC, 0, EVL_MUTEX_NORMAL|EVL_CLONE_PRIVATE, "Mutex 2");
    evl_create_mutex(&m3, EVL_CLOCK_MONOTONIC, 0, EVL_MUTEX_NORMAL|EVL_CLONE_PRIVATE, "Mutex 3");

    // Thread 1
    pthread_attr_t tattr;
    sched_param param;
    pthread_t tid;
    cpu_set_t tkcpu;
    CPU_ZERO(&tkcpu);
    CPU_SET(1, &tkcpu);
    pthread_attr_init(&tattr);
    pthread_attr_getschedparam(&tattr, &param);
    pthread_attr_setstacksize(&tattr, 1024*1024);
    pthread_attr_setaffinity_np(&tattr, sizeof(cpu_set_t), &tkcpu);
    pthread_attr_setschedpolicy(&tattr, SCHED_FIFO);
    param.sched_priority = 83;
    pthread_attr_setschedparam(&tattr, &param);
    pthread_attr_setinheritsched(&tattr, PTHREAD_EXPLICIT_SCHED);
    pthread_create(&tid, &tattr, Thread1, NULL);

    // Thread 2
    pthread_attr_t tattr2;
    sched_param param2;
    pthread_t tid2;
    cpu_set_t tkcpu2;
    CPU_ZERO(&tkcpu2);
    CPU_SET(2, &tkcpu2);
    pthread_attr_init(&tattr2);
    pthread_attr_getschedparam(&tattr2, &param2);
    pthread_attr_setstacksize(&tattr2, 1024*1024);
    pthread_attr_setaffinity_np(&tattr2, sizeof(cpu_set_t), &tkcpu2);
    pthread_attr_setschedpolicy(&tattr2, SCHED_FIFO);
    param2.sched_priority = 82;
    pthread_attr_setschedparam(&tattr2, &param2);
    pthread_attr_setinheritsched(&tattr2, PTHREAD_EXPLICIT_SCHED);
    pthread_create(&tid2, &tattr2, Thread2, NULL);

    // Thread 3
    pthread_attr_t tattr3;
    sched_param param3;
    pthread_t tid3;
    cpu_set_t tkcpu3;
    CPU_ZERO(&tkcpu3);
    CPU_SET(3, &tkcpu3);
    pthread_attr_init(&tattr3);
    pthread_attr_getschedparam(&tattr3, &param3);
    pthread_attr_setstacksize(&tattr3, 1024*1024);
    pthread_attr_setaffinity_np(&tattr3, sizeof(cpu_set_t), &tkcpu3);
    pthread_attr_setschedpolicy(&tattr3, SCHED_FIFO);
    param3.sched_priority = 81;
    pthread_attr_setschedparam(&tattr3, &param3);
    pthread_attr_setinheritsched(&tattr3, PTHREAD_EXPLICIT_SCHED);
    pthread_create(&tid3, &tattr3, Thread3, NULL);

    // Thread 4
    pthread_attr_t tattr4;
    sched_param param4;
    pthread_t tid4;
    cpu_set_t tkcpu4;
    CPU_ZERO(&tkcpu4);
    CPU_SET(4, &tkcpu4);
    pthread_attr_init(&tattr4);
    pthread_attr_getschedparam(&tattr4, &param4);
    pthread_attr_setstacksize(&tattr4, 1024*1024);
    pthread_attr_setaffinity_np(&tattr4, sizeof(cpu_set_t), &tkcpu4);
    pthread_attr_setschedpolicy(&tattr4, SCHED_FIFO);
    param4.sched_priority = 80;
    pthread_attr_setschedparam(&tattr4, &param4);
    pthread_attr_setinheritsched(&tattr4, PTHREAD_EXPLICIT_SCHED);
    pthread_create(&tid4, &tattr4, Thread4, NULL);

    // Do not exit so we can see stats
    while(true)
    {
        sleep(1);
    }

    return 0;
}

[-- Attachment #2: smime.p7s --]
[-- Type: application/pkcs7-signature, Size: 6759 bytes --]

^ permalink raw reply	[flat|nested] 26+ messages in thread