complete_all() with 5 waiters in swake_up_all_locked

complete_all() with 5 waiters in swake_up_all_locked

[Philipp K]

Hi,

I've been fuzzing linux-rt with syzkaller and found a way to trigger
$title warning.

The reproducer works with a defconfig + CONFIG_NET_KEY=y on 4.18.12-rt6 kernel.

Philipp

PS: Liebe Grüße von Lukas Bulwahn


Syzkaller hit 'WARNING in swake_up_all_locked' bug.

------------[ cut here ]------------
WARNING: CPU: 3 PID: 1778 at /kernel-source//kernel/sched/swait.c:48
swake_up_all_locked+0x18d/0x220 kernel/sched/swait.c:48
complete_all() with 5 waiters
Kernel panic - not syncing: panic_on_warn set ...

CPU: 3 PID: 1778 Comm: cryptomgr_probe Not tainted 4.9.56-rt37-hpad-a1-64-abl #1
Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS
Ubuntu-1.8.2-1ubuntu1 04/01/2014
 ffff88006a907ba8 ffffffff8b88c869 ffffffff8ca440a0 ffff88006a907c80
 ffffffff8ca58600 ffffffff8b01f84d ffff88006a907c70 ffffffff8b1f8ec9
 0000000041b58ab3 ffffffff8ceea20b ffffffff8b1f8d13 ffffffff0000001d
Call Trace:
 [<ffffffff8b88c869>] __dump_stack lib/dump_stack.c:15 [inline]
 [<ffffffff8b88c869>] dump_stack+0x6c/0x93 lib/dump_stack.c:51
 [<ffffffff8b1f8ec9>] panic+0x1b6/0x397 kernel/panic.c:179
 [<ffffffff8af32dd2>] __warn+0x1e2/0x200 kernel/panic.c:544
 [<ffffffff8af32eb7>] warn_slowpath_fmt+0xc7/0x100 kernel/panic.c:567
 [<ffffffff8b01f84d>] swake_up_all_locked+0x18d/0x220 kernel/sched/swait.c:48
 [<ffffffff8b0208dc>] complete_all+0x4c/0x70 kernel/sched/completion.c:55
 [<ffffffff8b7a95c3>] cryptomgr_probe+0x173/0x240 crypto/algboss.c:89
 [<ffffffff8af980cd>] kthread+0x22d/0x2b0 kernel/kthread.c:211
 [<ffffffff8c7f89a2>] ret_from_fork+0x22/0x30 arch/x86/entry/entry_64.S:433
Shutting down cpus with NMI
Dumping ftrace buffer:
   (ftrace buffer empty)
Kernel Offset: 0xac00000 from 0xffffffff80200000 (relocation range:
0xffffffff80000000-0xffffffffbfffffff)


Syzkaller reproducer:
# {Threaded:false Collide:false Repeat:true RepeatTimes:0 Procs:8
Sandbox:none Fault:false FaultCall:-1 FaultNth:0 EnableTun:false
UseTmpDir:false EnableCgroups:false EnableNetdev:false ResetNet:false
HandleSegv:false Repro:false Trace:false}
r0 = socket$key(0xf, 0x3, 0x2)
sendmsg$key(r0, &(0x7f0000000180)={0x0, 0x0,
&(0x7f0000000040)={&(0x7f0000000000)={0x2, 0x7, 0x0, 0x0, 0x2, 0x0,
0x4}, 0x10}}, 0x0)


C reproducer:
// autogenerated by syzkaller (https://github.com/google/syzkaller)

#define _GNU_SOURCE

#include <dirent.h>
#include <endian.h>
#include <errno.h>
#include <fcntl.h>
#include <sched.h>
#include <signal.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mount.h>
#include <sys/prctl.h>
#include <sys/resource.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <time.h>
#include <unistd.h>

unsigned long long procid;

static void sleep_ms(uint64_t ms)
{
    usleep(ms * 1000);
}

static uint64_t current_time_ms()
{
    struct timespec ts;
    if (clock_gettime(CLOCK_MONOTONIC, &ts))
    exit(1);
    return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 1000000;
}

static void setup_common()
{
    if (mount(0, "/sys/fs/fuse/connections", "fusectl", 0, 0)) {
    }
}

static void loop();

static void sandbox_common()
{
    prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0);
    setpgrp();
    setsid();
    struct rlimit rlim;
    rlim.rlim_cur = rlim.rlim_max = 160 << 20;
    setrlimit(RLIMIT_AS, &rlim);
    rlim.rlim_cur = rlim.rlim_max = 8 << 20;
    setrlimit(RLIMIT_MEMLOCK, &rlim);
    rlim.rlim_cur = rlim.rlim_max = 136 << 20;
    setrlimit(RLIMIT_FSIZE, &rlim);
    rlim.rlim_cur = rlim.rlim_max = 1 << 20;
    setrlimit(RLIMIT_STACK, &rlim);
    rlim.rlim_cur = rlim.rlim_max = 0;
    setrlimit(RLIMIT_CORE, &rlim);
    rlim.rlim_cur = rlim.rlim_max = 256;
    setrlimit(RLIMIT_NOFILE, &rlim);
    if (unshare(CLONE_NEWNS)) {
    }
    if (unshare(CLONE_NEWIPC)) {
    }
    if (unshare(0x02000000)) {
    }
    if (unshare(CLONE_NEWUTS)) {
    }
    if (unshare(CLONE_SYSVSEM)) {
    }
}

int wait_for_loop(int pid)
{
    if (pid < 0)
    exit(1);
    int status = 0;
    while (waitpid(-1, &status, __WALL) != pid) {
    }
    return WEXITSTATUS(status);
}

static int do_sandbox_none(void)
{
    if (unshare(CLONE_NEWPID)) {
    }
    int pid = fork();
    if (pid != 0)
        return wait_for_loop(pid);
    setup_common();
    sandbox_common();
    if (unshare(CLONE_NEWNET)) {
    }
    loop();
    exit(1);
}

static void kill_and_wait(int pid, int* status)
{
    kill(-pid, SIGKILL);
    kill(pid, SIGKILL);
    int i;
    for (i = 0; i < 100; i++) {
        if (waitpid(-1, status, WNOHANG | __WALL) == pid)
            return;
        usleep(1000);
    }
    DIR* dir = opendir("/sys/fs/fuse/connections");
    if (dir) {
        for (;;) {
            struct dirent* ent = readdir(dir);
            if (!ent)
                break;
            if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0)
                continue;
            char abort[300];
            snprintf(abort, sizeof(abort),
"/sys/fs/fuse/connections/%s/abort", ent->d_name);
            int fd = open(abort, O_WRONLY);
            if (fd == -1) {
                continue;
            }
            if (write(fd, abort, 1) < 0) {
            }
            close(fd);
        }
        closedir(dir);
    } else {
    }
    while (waitpid(-1, status, __WALL) != pid) {
    }
}

#define SYZ_HAVE_SETUP_TEST 1
static void setup_test()
{
    prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0);
    setpgrp();
}

#define SYZ_HAVE_RESET_TEST 1
static void reset_test()
{
    int fd;
    for (fd = 3; fd < 30; fd++)
        close(fd);
}

static void execute_one();

#define WAIT_FLAGS __WALL

static void loop()
{
    int iter;
    for (iter = 0;; iter++) {
        int pid = fork();
        if (pid < 0)
    exit(1);
        if (pid == 0) {
            setup_test();
            execute_one();
            reset_test();
            exit(0);
        }
        int status = 0;
        uint64_t start = current_time_ms();
        for (;;) {
            if (waitpid(-1, &status, WNOHANG | WAIT_FLAGS) == pid)
                break;
            sleep_ms(1);
            if (current_time_ms() - start < 5 * 1000)
                continue;
            kill_and_wait(pid, &status);
            break;
        }
    }
}

uint64_t r[1] = {0xffffffffffffffff};

void execute_one()
{
        long res = 0;
    res = syscall(__NR_socket, 0xf, 3, 2);  // socket(PF_KEY,
SOCK_RAW, AF_INET);
    if (res != -1)
        r[0] = res;
*(uint64_t*)0x20000180 = 0;
*(uint32_t*)0x20000188 = 0;
*(uint64_t*)0x20000190 = 0x20000040;
*(uint64_t*)0x20000040 = 0x20000000;
*(uint8_t*)0x20000000 = 2;
*(uint8_t*)0x20000001 = 7;
*(uint8_t*)0x20000002 = 0;
*(uint8_t*)0x20000003 = 0;
*(uint16_t*)0x20000004 = 2;
*(uint16_t*)0x20000006 = 0;
*(uint32_t*)0x20000008 = 4;
*(uint32_t*)0x2000000c = 0;
*(uint64_t*)0x20000048 = 0x10;
*(uint64_t*)0x20000198 = 1;
*(uint64_t*)0x200001a0 = 0;
*(uint64_t*)0x200001a8 = 0;
*(uint32_t*)0x200001b0 = 0;
    syscall(__NR_sendmsg, r[0], 0x20000180, 0);

}
int main()
{
        syscall(__NR_mmap, 0x20000000, 0x1000000, 3, 0x32, -1, 0);
    for (procid = 0; procid < 8; procid++) {
        if (fork() == 0) {
            do_sandbox_none();
        }
    }
    sleep(1000000);
    return 0;
}

Re: complete_all() with 5 waiters in swake_up_all_locked

[Sebastian Andrzej Siewior]

On 2018-10-10 11:15:46 [+0200], Philipp K wrote:
> Hi,
Hi,

sorry for being late.

> I've been fuzzing linux-rt with syzkaller and found a way to trigger
> $title warning.
> 
> The reproducer works with a defconfig + CONFIG_NET_KEY=y on 4.18.12-rt6 kernel.

I didn't manage to reproduce this on v4.19-RT. Also that backtrace is
from v4.9. With that test case I didn't manage to end up in
cryptomgr_probe() which would be a requirement.

So. The warning says that the system managed to wake up 5 tasks in one
go, that is without re-enabling the interrupts. It has been agreed that
two wakes are okay and more should be avoided. The problem is that it
may cause latency spikes if for some reason the system wakes say 100
tasks like that.
We have one exception for the PM case because nobody does (or should do)
suspend-to-ram and run a RT task which has to meet a deadline.

So for this case, we should figure out why it managed to wake 5 tasks
and how bad can it get and can we limit it somehow.

> Philipp
> 
> PS: Liebe Grüße von Lukas Bulwahn
Ah. Greetings!

Sebastian