All of lore.kernel.org
 help / color / mirror / Atom feed
* Re: UST Hanging at Tracepoint
       [not found] <3F56B905CF2083408E8482044F20428AEF604D22@ONWVEXCHMB01.ciena.com>
@ 2012-06-08 20:29 ` Burton, Michael
       [not found] ` <3F56B905CF2083408E8482044F20428AEF6A2DBB@ONWVEXCHMB01.ciena.com>
       [not found] ` <3F56B905CF2083408E8482044F20428AEF78BC79@ONWVEXCHMB01.ciena.com>
  2 siblings, 0 replies; 16+ messages in thread
From: Burton, Michael @ 2012-06-08 20:29 UTC (permalink / raw)
  To: lttng-dev


[-- Attachment #1.1: Type: text/plain, Size: 1071 bytes --]

For further insight, I've attached the strace from the hanging program.  I also upgraded LTTng-UST to 2.0.3 and userspace-rcu to 0.7.3.

Michael

From: Burton, Michael [mailto:mburton@ciena.com]
Sent: Thursday, June 07, 2012 2:16 PM
To: lttng-dev@lists.lttng.org
Subject: [lttng-dev] UST Hanging at Tracepoint

I am working on getting LTTng 2.0 working in our 2.6.35 kernel.  I have kernel tracing working but I'm having problems with UST.

I have a program with a dynamic UST tracepoint.  When I run the program with all UST events enabled (lttng enable-event -a -u) the program hangs on the tracepoint.  The last line from the UST debug is:

libust[6184/6185]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)

Any insight into why this is happening?  I've attached the UST and lttng-sessiond debug generated by the program.

I am running the following:
lttng-modules-2.6.32 (found through this mailing list)
lttng-tools-2.0.1
lttng-ust-2.0.2
userspace-rcu-0.7.0

Thanks,
Michael

[-- Attachment #1.2: Type: text/html, Size: 6782 bytes --]

[-- Attachment #2: strace_debug.txt --]
[-- Type: text/plain, Size: 12060 bytes --]

execve("/ciena/bin/idp", ["idp", "help"], [/* 11 vars */]) = 0
brk(0)                                  = 0x804e000
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb783c000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat64(3, {st_mode=S_IFREG|0644, st_size=17420, ...}) = 0
mmap2(NULL, 17420, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7837000
close(3)                                = 0
open("/usr/lib/liblttng-ust.so.0", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220A\0\0004\0\0\0$"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=209876, ...}) = 0
mmap2(NULL, 212940, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7803000
mmap2(0xb7832000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2e) = 0xb7832000
close(3)                                = 0
open("/usr/lib/liblttng-ust-ctl.so.0", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200-\0\0004\0\0\0\274"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=140020, ...}) = 0
mmap2(NULL, 142828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77e0000
mmap2(0xb7802000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21) = 0xb7802000
close(3)                                = 0
open("/usr/lib/liblttng-ust-fork.so.0", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\5\0\0004\0\0\0X"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=3864, ...}) = 0
mmap2(NULL, 6820, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77de000
mmap2(0xb77df000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77df000
close(3)                                = 0
open("/usr/lib/liblttng-ust-tracepoint.so.0", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220\v\0\0004\0\0\0008"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=16632, ...}) = 0
mmap2(NULL, 19944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d9000
mmap2(0xb77dd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xb77dd000
close(3)                                = 0
open("/usr/lib/liblttng-ust-libc-wrapper.so.0", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\t\0\0004\0\0\0\364"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=9300, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77d8000
mmap2(NULL, 12104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d5000
mmap2(0xb77d7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77d7000
close(3)                                = 0
open("/usr/lib/liburcu.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d0000
mmap2(0xb77d4000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77d4000
close(3)                                = 0
open("/usr/lib/liburcu-common.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\5\0\0004\0\0\0\34"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=5084, ...}) = 0
mmap2(NULL, 8032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ce000
mmap2(0xb77cf000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77cf000
close(3)                                = 0
open("/usr/lib/liburcu-qsbr.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\22\0\0004\0\0\0\374"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c9000
mmap2(0xb77cd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77cd000
close(3)                                = 0
open("/usr/lib/liburcu-mb.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c4000
mmap2(0xb77c8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c8000
close(3)                                = 0
open("/usr/lib/liburcu-signal.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\23\0\0004\0\0\0\10"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=18712, ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77c3000
mmap2(NULL, 21704, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77bd000
mmap2(0xb77c2000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c2000
close(3)                                = 0
open("/usr/lib/liburcu-cds.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\f\0\0004\0\0\0t"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=26204, ...}) = 0
mmap2(NULL, 25004, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b6000
mmap2(0xb77bc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb77bc000
close(3)                                = 0
open("/usr/lib/liburcu-bp.so.1", O_RDONLY) = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\24\0\0004\0\0\0\360"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=19968, ...}) = 0
mmap2(NULL, 23128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b0000
mmap2(0xb77b5000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77b5000
close(3)                                = 0
open("/lib/libdl.so.2", O_RDONLY)       = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \n\0\0004\0\0\0<"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=9668, ...}) = 0
mmap2(NULL, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ac000
mmap2(0xb77ae000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77ae000
close(3)                                = 0
open("/lib/libuuid.so.1", O_RDONLY)     = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@\r\0\0004\0\0\0\350"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=12872, ...}) = 0
mmap2(NULL, 15632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77a8000
mmap2(0xb77ab000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2) = 0xb77ab000
close(3)                                = 0
open("/lib/libc.so.6", O_RDONLY)        = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220n\1\0004\0\0\0L"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=1347780, ...}) = 0
mmap2(NULL, 1358120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb765c000
mprotect(0xb77a1000, 4096, PROT_NONE)   = 0
mmap2(0xb77a2000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x145) = 0xb77a2000
mmap2(0xb77a5000, 10536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb77a5000
close(3)                                = 0
open("/lib/librt.so.1", O_RDONLY)       = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\30\0\0004\0\0\0\230"..., 512) = 512
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb765b000
fstat64(3, {st_mode=S_IFREG|0755, st_size=30616, ...}) = 0
mmap2(NULL, 33364, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7652000
mmap2(0xb7659000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7659000
close(3)                                = 0
open("/lib/libpthread.so.0", O_RDONLY)  = 3
read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360J\0\0004\0\0\0\354"..., 512) = 512
fstat64(3, {st_mode=S_IFREG|0755, st_size=88420, ...}) = 0
mmap2(NULL, 98828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7639000
mmap2(0xb764e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0xb764e000
mmap2(0xb7650000, 4620, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7650000
close(3)                                = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7638000
mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7636000
set_thread_area({entry_number:-1 -> 6, base_addr:0xb7636d80, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
mprotect(0xb764e000, 4096, PROT_READ)   = 0
mprotect(0xb7659000, 4096, PROT_READ)   = 0
mprotect(0xb77a2000, 8192, PROT_READ)   = 0
mprotect(0xb77ae000, 4096, PROT_READ)   = 0
mprotect(0xb785a000, 4096, PROT_READ)   = 0
munmap(0xb7837000, 17420)               = 0
set_tid_address(0xb7636de8)             = 3050
set_robust_list(0xb7636df0, 0xc)        = 0
futex(0xbfaaf560, FUTEX_WAKE_PRIVATE, 1) = 0
futex(0xbfaaf560, 0x189 /* FUTEX_??? */, 1, NULL, bfaaf570) = -1 EAGAIN (Resource temporarily unavailable)
rt_sigaction(SIGRTMIN, {0xb763d4f0, [], SA_SIGINFO}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0xb763d9c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
uname({sys="Linux", node="5410", ...})  = 0
rt_sigaction(SIGUSR1, {0xb77bec0a, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
futex(0xb77af06c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
brk(0)                                  = 0x804e000
brk(0x806f000)                          = 0x806f000
gettimeofday({1339187216, 880428}, NULL) = 0
open("/dev/urandom", O_RDONLY|O_LARGEFILE) = 3
fcntl64(3, F_GETFD)                     = 0
fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
getuid32()                              = 0
gettimeofday({1339187216, 889603}, NULL) = 0
gettimeofday({1339187216, 894471}, NULL) = 0
read(3, "\275\0051F\215\373\371\202\377kfb/\362\303N"..., 16) = 16
clock_gettime(CLOCK_REALTIME, {1339187216, 902329405}) = 0
getuid32()                              = 0
geteuid32()                             = 0
rt_sigprocmask(SIG_SETMASK, ~[RTMIN RT_1], [], 8) = 0
mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6e36000
mprotect(0xb6e36000, 4096, PROT_NONE)   = 0
clone(child_stack=0xb7634d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb7635bd8, {entry_number:6, base_addr:0xb7635b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb7635bd8) = 3051
mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6636000
mprotect(0xb6636000, 4096, PROT_NONE)   = 0
clone(child_stack=0xb6e34d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb6e35bd8, {entry_number:6, base_addr:0xb6e35b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb6e35bd8) = 3052
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
gettimeofday({1339187216, 974869}, NULL) = 0
futex(0xb7836e30, FUTEX_WAIT_PRIVATE, 0, {2, 927460405}) = -1 ETIMEDOUT (Connection timed out)
gettid()                                = 3050
write(2, "libust[3050/3050]: Error: Timed o"..., 107libust[3050/3050]: Error: Timed out waiting for ltt-sessiond (in lttng_ust_init() at lttng-ust-comm.c:912)
) = 107
futex(0xb7836e80, FUTEX_WAIT_PRIVATE, 2, NULL

[-- Attachment #3: Type: text/plain, Size: 155 bytes --]

_______________________________________________
lttng-dev mailing list
lttng-dev@lists.lttng.org
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

^ permalink raw reply	[flat|nested] 16+ messages in thread
* Re: UST Hanging at Tracepoint
       [not found] ` <3F56B905CF2083408E8482044F20428AEF6A2DBB@ONWVEXCHMB01.ciena.com>
@ 2012-06-09  1:38   ` Mathieu Desnoyers
       [not found]   ` <20120609013854.GB10978@Krystal>
  1 sibling, 0 replies; 16+ messages in thread
From: Mathieu Desnoyers @ 2012-06-09  1:38 UTC (permalink / raw)
  To: Burton, Michael; +Cc: lttng-dev

* Burton, Michael (mburton@ciena.com) wrote:
> For further insight, I've attached the strace from the hanging
> program.  I also upgraded LTTng-UST to 2.0.3 and userspace-rcu to
> 0.7.3.

Haven't had time to look at it yet, but could you also provide the
output of:

LTTNG_UST_DEBUG=1 youprogname

(starting your UST instrumented program with LTTNG_UST_DEBUG=1 env. var.
set)

Thanks!

Mathieu

> 
> Michael
> 
> From: Burton, Michael [mailto:mburton@ciena.com]
> Sent: Thursday, June 07, 2012 2:16 PM
> To: lttng-dev@lists.lttng.org
> Subject: [lttng-dev] UST Hanging at Tracepoint
> 
> I am working on getting LTTng 2.0 working in our 2.6.35 kernel.  I have kernel tracing working but I'm having problems with UST.
> 
> I have a program with a dynamic UST tracepoint.  When I run the program with all UST events enabled (lttng enable-event -a -u) the program hangs on the tracepoint.  The last line from the UST debug is:
> 
> libust[6184/6185]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)
> 
> Any insight into why this is happening?  I've attached the UST and lttng-sessiond debug generated by the program.
> 
> I am running the following:
> lttng-modules-2.6.32 (found through this mailing list)
> lttng-tools-2.0.1
> lttng-ust-2.0.2
> userspace-rcu-0.7.0
> 
> Thanks,
> Michael

Content-Description: strace_debug.txt
> execve("/ciena/bin/idp", ["idp", "help"], [/* 11 vars */]) = 0
> brk(0)                                  = 0x804e000
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb783c000
> access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
> open("/etc/ld.so.cache", O_RDONLY)      = 3
> fstat64(3, {st_mode=S_IFREG|0644, st_size=17420, ...}) = 0
> mmap2(NULL, 17420, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7837000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220A\0\0004\0\0\0$"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=209876, ...}) = 0
> mmap2(NULL, 212940, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7803000
> mmap2(0xb7832000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2e) = 0xb7832000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust-ctl.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200-\0\0004\0\0\0\274"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=140020, ...}) = 0
> mmap2(NULL, 142828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77e0000
> mmap2(0xb7802000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21) = 0xb7802000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust-fork.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\5\0\0004\0\0\0X"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=3864, ...}) = 0
> mmap2(NULL, 6820, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77de000
> mmap2(0xb77df000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77df000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust-tracepoint.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220\v\0\0004\0\0\0008"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=16632, ...}) = 0
> mmap2(NULL, 19944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d9000
> mmap2(0xb77dd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xb77dd000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust-libc-wrapper.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\t\0\0004\0\0\0\364"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=9300, ...}) = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77d8000
> mmap2(NULL, 12104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d5000
> mmap2(0xb77d7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77d7000
> close(3)                                = 0
> open("/usr/lib/liburcu.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d0000
> mmap2(0xb77d4000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77d4000
> close(3)                                = 0
> open("/usr/lib/liburcu-common.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\5\0\0004\0\0\0\34"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=5084, ...}) = 0
> mmap2(NULL, 8032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ce000
> mmap2(0xb77cf000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77cf000
> close(3)                                = 0
> open("/usr/lib/liburcu-qsbr.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\22\0\0004\0\0\0\374"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c9000
> mmap2(0xb77cd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77cd000
> close(3)                                = 0
> open("/usr/lib/liburcu-mb.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c4000
> mmap2(0xb77c8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c8000
> close(3)                                = 0
> open("/usr/lib/liburcu-signal.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\23\0\0004\0\0\0\10"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=18712, ...}) = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77c3000
> mmap2(NULL, 21704, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77bd000
> mmap2(0xb77c2000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c2000
> close(3)                                = 0
> open("/usr/lib/liburcu-cds.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\f\0\0004\0\0\0t"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=26204, ...}) = 0
> mmap2(NULL, 25004, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b6000
> mmap2(0xb77bc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb77bc000
> close(3)                                = 0
> open("/usr/lib/liburcu-bp.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\24\0\0004\0\0\0\360"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=19968, ...}) = 0
> mmap2(NULL, 23128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b0000
> mmap2(0xb77b5000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77b5000
> close(3)                                = 0
> open("/lib/libdl.so.2", O_RDONLY)       = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \n\0\0004\0\0\0<"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=9668, ...}) = 0
> mmap2(NULL, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ac000
> mmap2(0xb77ae000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77ae000
> close(3)                                = 0
> open("/lib/libuuid.so.1", O_RDONLY)     = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@\r\0\0004\0\0\0\350"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=12872, ...}) = 0
> mmap2(NULL, 15632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77a8000
> mmap2(0xb77ab000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2) = 0xb77ab000
> close(3)                                = 0
> open("/lib/libc.so.6", O_RDONLY)        = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220n\1\0004\0\0\0L"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=1347780, ...}) = 0
> mmap2(NULL, 1358120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb765c000
> mprotect(0xb77a1000, 4096, PROT_NONE)   = 0
> mmap2(0xb77a2000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x145) = 0xb77a2000
> mmap2(0xb77a5000, 10536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb77a5000
> close(3)                                = 0
> open("/lib/librt.so.1", O_RDONLY)       = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\30\0\0004\0\0\0\230"..., 512) = 512
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb765b000
> fstat64(3, {st_mode=S_IFREG|0755, st_size=30616, ...}) = 0
> mmap2(NULL, 33364, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7652000
> mmap2(0xb7659000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7659000
> close(3)                                = 0
> open("/lib/libpthread.so.0", O_RDONLY)  = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360J\0\0004\0\0\0\354"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=88420, ...}) = 0
> mmap2(NULL, 98828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7639000
> mmap2(0xb764e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0xb764e000
> mmap2(0xb7650000, 4620, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7650000
> close(3)                                = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7638000
> mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7636000
> set_thread_area({entry_number:-1 -> 6, base_addr:0xb7636d80, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
> mprotect(0xb764e000, 4096, PROT_READ)   = 0
> mprotect(0xb7659000, 4096, PROT_READ)   = 0
> mprotect(0xb77a2000, 8192, PROT_READ)   = 0
> mprotect(0xb77ae000, 4096, PROT_READ)   = 0
> mprotect(0xb785a000, 4096, PROT_READ)   = 0
> munmap(0xb7837000, 17420)               = 0
> set_tid_address(0xb7636de8)             = 3050
> set_robust_list(0xb7636df0, 0xc)        = 0
> futex(0xbfaaf560, FUTEX_WAKE_PRIVATE, 1) = 0
> futex(0xbfaaf560, 0x189 /* FUTEX_??? */, 1, NULL, bfaaf570) = -1 EAGAIN (Resource temporarily unavailable)
> rt_sigaction(SIGRTMIN, {0xb763d4f0, [], SA_SIGINFO}, NULL, 8) = 0
> rt_sigaction(SIGRT_1, {0xb763d9c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
> uname({sys="Linux", node="5410", ...})  = 0
> rt_sigaction(SIGUSR1, {0xb77bec0a, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> futex(0xb77af06c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
> brk(0)                                  = 0x804e000
> brk(0x806f000)                          = 0x806f000
> gettimeofday({1339187216, 880428}, NULL) = 0
> open("/dev/urandom", O_RDONLY|O_LARGEFILE) = 3
> fcntl64(3, F_GETFD)                     = 0
> fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
> getuid32()                              = 0
> gettimeofday({1339187216, 889603}, NULL) = 0
> gettimeofday({1339187216, 894471}, NULL) = 0
> read(3, "\275\0051F\215\373\371\202\377kfb/\362\303N"..., 16) = 16
> clock_gettime(CLOCK_REALTIME, {1339187216, 902329405}) = 0
> getuid32()                              = 0
> geteuid32()                             = 0
> rt_sigprocmask(SIG_SETMASK, ~[RTMIN RT_1], [], 8) = 0
> mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6e36000
> mprotect(0xb6e36000, 4096, PROT_NONE)   = 0
> clone(child_stack=0xb7634d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb7635bd8, {entry_number:6, base_addr:0xb7635b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb7635bd8) = 3051
> mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6636000
> mprotect(0xb6636000, 4096, PROT_NONE)   = 0
> clone(child_stack=0xb6e34d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb6e35bd8, {entry_number:6, base_addr:0xb6e35b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb6e35bd8) = 3052
> rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
> gettimeofday({1339187216, 974869}, NULL) = 0
> futex(0xb7836e30, FUTEX_WAIT_PRIVATE, 0, {2, 927460405}) = -1 ETIMEDOUT (Connection timed out)
> gettid()                                = 3050
> write(2, "libust[3050/3050]: Error: Timed o"..., 107libust[3050/3050]: Error: Timed out waiting for ltt-sessiond (in lttng_ust_init() at lttng-ust-comm.c:912)
> ) = 107
> futex(0xb7836e80, FUTEX_WAIT_PRIVATE, 2, NULL
> _______________________________________________
> lttng-dev mailing list
> lttng-dev@lists.lttng.org
> http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev


-- 
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

^ permalink raw reply	[flat|nested] 16+ messages in thread
* Re: UST Hanging at Tracepoint
       [not found]   ` <20120609013854.GB10978@Krystal>
@ 2012-06-11 15:08     ` Burton, Michael
  2012-06-12  2:48     ` Burton, Michael
  1 sibling, 0 replies; 16+ messages in thread
From: Burton, Michael @ 2012-06-11 15:08 UTC (permalink / raw)
  To: Mathieu Desnoyers; +Cc: lttng-dev

Here is the output from UST debug:

liblttng_ust_tracepoint[1022/1022]: just registered a tracepoints section from 0xb76f9e04 and having 1 tracepoints (in tracepoint_register_lib() at tracepoint.c:639)
liblttng_ust_tracepoint[1022/1022]: registered tracepoint: lttng_ust:metadata (in tracepoint_register_lib() at tracepoint.c:644)
libust[1022/1022]: LTT : ltt ring buffer client init
 (in ltt_ring_buffer_metadata_client_init() at ltt-ring-buffer-metadata-client.h:334)
libust[1022/1022]: LTT : ltt ring buffer client init
 (in ltt_ring_buffer_client_overwrite_init() at ltt-ring-buffer-client.h:584)
libust[1022/1022]: LTT : ltt ring buffer client init
 (in ltt_ring_buffer_client_discard_init() at ltt-ring-buffer-client.h:584)
libust[1022/1024]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[1022/1024]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[1022/1024]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[1022/1024]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[1022/1024]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[1022/1024]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[1022/1024]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[1022/1024]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[1022/1024]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[1022/1024]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[1022/1023]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)

-----Original Message-----
From: Mathieu Desnoyers [mailto:mathieu.desnoyers@efficios.com]
Sent: Friday, June 08, 2012 9:39 PM
To: Burton, Michael
Cc: lttng-dev@lists.lttng.org
Subject: Re: [lttng-dev] UST Hanging at Tracepoint

* Burton, Michael (mburton@ciena.com) wrote:
> For further insight, I've attached the strace from the hanging
> program.  I also upgraded LTTng-UST to 2.0.3 and userspace-rcu to
> 0.7.3.

Haven't had time to look at it yet, but could you also provide the
output of:

LTTNG_UST_DEBUG=1 youprogname

(starting your UST instrumented program with LTTNG_UST_DEBUG=1 env. var.
set)

Thanks!

Mathieu

>
> Michael
>
> From: Burton, Michael [mailto:mburton@ciena.com]
> Sent: Thursday, June 07, 2012 2:16 PM
> To: lttng-dev@lists.lttng.org
> Subject: [lttng-dev] UST Hanging at Tracepoint
>
> I am working on getting LTTng 2.0 working in our 2.6.35 kernel.  I have kernel tracing working but I'm having problems with UST.
>
> I have a program with a dynamic UST tracepoint.  When I run the program with all UST events enabled (lttng enable-event -a -u) the program hangs on the tracepoint.  The last line from the UST debug is:
>
> libust[6184/6185]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)
>
> Any insight into why this is happening?  I've attached the UST and lttng-sessiond debug generated by the program.
>
> I am running the following:
> lttng-modules-2.6.32 (found through this mailing list)
> lttng-tools-2.0.1
> lttng-ust-2.0.2
> userspace-rcu-0.7.0
>
> Thanks,
> Michael

Content-Description: strace_debug.txt
> execve("/ciena/bin/idp", ["idp", "help"], [/* 11 vars */]) = 0
> brk(0)                                  = 0x804e000
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb783c000
> access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
> open("/etc/ld.so.cache", O_RDONLY)      = 3
> fstat64(3, {st_mode=S_IFREG|0644, st_size=17420, ...}) = 0
> mmap2(NULL, 17420, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7837000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220A\0\0004\0\0\0$"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=209876, ...}) = 0
> mmap2(NULL, 212940, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7803000
> mmap2(0xb7832000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2e) = 0xb7832000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust-ctl.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200-\0\0004\0\0\0\274"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=140020, ...}) = 0
> mmap2(NULL, 142828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77e0000
> mmap2(0xb7802000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21) = 0xb7802000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust-fork.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\5\0\0004\0\0\0X"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=3864, ...}) = 0
> mmap2(NULL, 6820, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77de000
> mmap2(0xb77df000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77df000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust-tracepoint.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220\v\0\0004\0\0\0008"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=16632, ...}) = 0
> mmap2(NULL, 19944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d9000
> mmap2(0xb77dd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xb77dd000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust-libc-wrapper.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\t\0\0004\0\0\0\364"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=9300, ...}) = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77d8000
> mmap2(NULL, 12104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d5000
> mmap2(0xb77d7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77d7000
> close(3)                                = 0
> open("/usr/lib/liburcu.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d0000
> mmap2(0xb77d4000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77d4000
> close(3)                                = 0
> open("/usr/lib/liburcu-common.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\5\0\0004\0\0\0\34"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=5084, ...}) = 0
> mmap2(NULL, 8032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ce000
> mmap2(0xb77cf000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77cf000
> close(3)                                = 0
> open("/usr/lib/liburcu-qsbr.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\22\0\0004\0\0\0\374"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c9000
> mmap2(0xb77cd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77cd000
> close(3)                                = 0
> open("/usr/lib/liburcu-mb.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c4000
> mmap2(0xb77c8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c8000
> close(3)                                = 0
> open("/usr/lib/liburcu-signal.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\23\0\0004\0\0\0\10"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=18712, ...}) = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77c3000
> mmap2(NULL, 21704, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77bd000
> mmap2(0xb77c2000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c2000
> close(3)                                = 0
> open("/usr/lib/liburcu-cds.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\f\0\0004\0\0\0t"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=26204, ...}) = 0
> mmap2(NULL, 25004, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b6000
> mmap2(0xb77bc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb77bc000
> close(3)                                = 0
> open("/usr/lib/liburcu-bp.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\24\0\0004\0\0\0\360"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=19968, ...}) = 0
> mmap2(NULL, 23128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b0000
> mmap2(0xb77b5000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77b5000
> close(3)                                = 0
> open("/lib/libdl.so.2", O_RDONLY)       = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \n\0\0004\0\0\0<"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=9668, ...}) = 0
> mmap2(NULL, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ac000
> mmap2(0xb77ae000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77ae000
> close(3)                                = 0
> open("/lib/libuuid.so.1", O_RDONLY)     = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@\r\0\0004\0\0\0\350"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=12872, ...}) = 0
> mmap2(NULL, 15632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77a8000
> mmap2(0xb77ab000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2) = 0xb77ab000
> close(3)                                = 0
> open("/lib/libc.so.6", O_RDONLY)        = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220n\1\0004\0\0\0L"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=1347780, ...}) = 0
> mmap2(NULL, 1358120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb765c000
> mprotect(0xb77a1000, 4096, PROT_NONE)   = 0
> mmap2(0xb77a2000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x145) = 0xb77a2000
> mmap2(0xb77a5000, 10536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb77a5000
> close(3)                                = 0
> open("/lib/librt.so.1", O_RDONLY)       = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\30\0\0004\0\0\0\230"..., 512) = 512
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb765b000
> fstat64(3, {st_mode=S_IFREG|0755, st_size=30616, ...}) = 0
> mmap2(NULL, 33364, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7652000
> mmap2(0xb7659000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7659000
> close(3)                                = 0
> open("/lib/libpthread.so.0", O_RDONLY)  = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360J\0\0004\0\0\0\354"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=88420, ...}) = 0
> mmap2(NULL, 98828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7639000
> mmap2(0xb764e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0xb764e000
> mmap2(0xb7650000, 4620, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7650000
> close(3)                                = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7638000
> mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7636000
> set_thread_area({entry_number:-1 -> 6, base_addr:0xb7636d80, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
> mprotect(0xb764e000, 4096, PROT_READ)   = 0
> mprotect(0xb7659000, 4096, PROT_READ)   = 0
> mprotect(0xb77a2000, 8192, PROT_READ)   = 0
> mprotect(0xb77ae000, 4096, PROT_READ)   = 0
> mprotect(0xb785a000, 4096, PROT_READ)   = 0
> munmap(0xb7837000, 17420)               = 0
> set_tid_address(0xb7636de8)             = 3050
> set_robust_list(0xb7636df0, 0xc)        = 0
> futex(0xbfaaf560, FUTEX_WAKE_PRIVATE, 1) = 0
> futex(0xbfaaf560, 0x189 /* FUTEX_??? */, 1, NULL, bfaaf570) = -1 EAGAIN (Resource temporarily unavailable)
> rt_sigaction(SIGRTMIN, {0xb763d4f0, [], SA_SIGINFO}, NULL, 8) = 0
> rt_sigaction(SIGRT_1, {0xb763d9c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
> uname({sys="Linux", node="5410", ...})  = 0
> rt_sigaction(SIGUSR1, {0xb77bec0a, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> futex(0xb77af06c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
> brk(0)                                  = 0x804e000
> brk(0x806f000)                          = 0x806f000
> gettimeofday({1339187216, 880428}, NULL) = 0
> open("/dev/urandom", O_RDONLY|O_LARGEFILE) = 3
> fcntl64(3, F_GETFD)                     = 0
> fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
> getuid32()                              = 0
> gettimeofday({1339187216, 889603}, NULL) = 0
> gettimeofday({1339187216, 894471}, NULL) = 0
> read(3, "\275\0051F\215\373\371\202\377kfb/\362\303N"..., 16) = 16
> clock_gettime(CLOCK_REALTIME, {1339187216, 902329405}) = 0
> getuid32()                              = 0
> geteuid32()                             = 0
> rt_sigprocmask(SIG_SETMASK, ~[RTMIN RT_1], [], 8) = 0
> mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6e36000
> mprotect(0xb6e36000, 4096, PROT_NONE)   = 0
> clone(child_stack=0xb7634d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb7635bd8, {entry_number:6, base_addr:0xb7635b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb7635bd8) = 3051
> mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6636000
> mprotect(0xb6636000, 4096, PROT_NONE)   = 0
> clone(child_stack=0xb6e34d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb6e35bd8, {entry_number:6, base_addr:0xb6e35b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb6e35bd8) = 3052
> rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
> gettimeofday({1339187216, 974869}, NULL) = 0
> futex(0xb7836e30, FUTEX_WAIT_PRIVATE, 0, {2, 927460405}) = -1 ETIMEDOUT (Connection timed out)
> gettid()                                = 3050
> write(2, "libust[3050/3050]: Error: Timed o"..., 107libust[3050/3050]: Error: Timed out waiting for ltt-sessiond (in lttng_ust_init() at lttng-ust-comm.c:912)
> ) = 107
> futex(0xb7836e80, FUTEX_WAIT_PRIVATE, 2, NULL
> _______________________________________________
> lttng-dev mailing list
> lttng-dev@lists.lttng.org
> http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev


--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

^ permalink raw reply	[flat|nested] 16+ messages in thread
* Re: UST Hanging at Tracepoint
       [not found]   ` <20120609013854.GB10978@Krystal>
  2012-06-11 15:08     ` Burton, Michael
@ 2012-06-12  2:48     ` Burton, Michael
  1 sibling, 0 replies; 16+ messages in thread
From: Burton, Michael @ 2012-06-12  2:48 UTC (permalink / raw)
  To: Mathieu Desnoyers; +Cc: lttng-dev

Mathieu,

I figured out the problem.  When we start sessiond on our system as root, the /.lttng/ folder is not created, along with its contents.  UST timed out since there was no sock_path.  I was able to get UST working once I got sessiond running.

Thanks.
Michael

________________________________________
From: Mathieu Desnoyers [mathieu.desnoyers@efficios.com]
Sent: 08 June 2012 21:38
To: Burton, Michael
Cc: lttng-dev@lists.lttng.org
Subject: Re: [lttng-dev] UST Hanging at Tracepoint

* Burton, Michael (mburton@ciena.com) wrote:
> For further insight, I've attached the strace from the hanging
> program.  I also upgraded LTTng-UST to 2.0.3 and userspace-rcu to
> 0.7.3.

Haven't had time to look at it yet, but could you also provide the
output of:

LTTNG_UST_DEBUG=1 youprogname

(starting your UST instrumented program with LTTNG_UST_DEBUG=1 env. var.
set)

Thanks!

Mathieu

>
> Michael
>
> From: Burton, Michael [mailto:mburton@ciena.com]
> Sent: Thursday, June 07, 2012 2:16 PM
> To: lttng-dev@lists.lttng.org
> Subject: [lttng-dev] UST Hanging at Tracepoint
>
> I am working on getting LTTng 2.0 working in our 2.6.35 kernel.  I have kernel tracing working but I'm having problems with UST.
>
> I have a program with a dynamic UST tracepoint.  When I run the program with all UST events enabled (lttng enable-event -a -u) the program hangs on the tracepoint.  The last line from the UST debug is:
>
> libust[6184/6185]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)
>
> Any insight into why this is happening?  I've attached the UST and lttng-sessiond debug generated by the program.
>
> I am running the following:
> lttng-modules-2.6.32 (found through this mailing list)
> lttng-tools-2.0.1
> lttng-ust-2.0.2
> userspace-rcu-0.7.0
>
> Thanks,
> Michael

Content-Description: strace_debug.txt
> execve("/ciena/bin/idp", ["idp", "help"], [/* 11 vars */]) = 0
> brk(0)                                  = 0x804e000
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb783c000
> access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
> open("/etc/ld.so.cache", O_RDONLY)      = 3
> fstat64(3, {st_mode=S_IFREG|0644, st_size=17420, ...}) = 0
> mmap2(NULL, 17420, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7837000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220A\0\0004\0\0\0$"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=209876, ...}) = 0
> mmap2(NULL, 212940, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7803000
> mmap2(0xb7832000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2e) = 0xb7832000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust-ctl.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200-\0\0004\0\0\0\274"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=140020, ...}) = 0
> mmap2(NULL, 142828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77e0000
> mmap2(0xb7802000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21) = 0xb7802000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust-fork.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\5\0\0004\0\0\0X"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=3864, ...}) = 0
> mmap2(NULL, 6820, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77de000
> mmap2(0xb77df000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77df000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust-tracepoint.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220\v\0\0004\0\0\0008"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=16632, ...}) = 0
> mmap2(NULL, 19944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d9000
> mmap2(0xb77dd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xb77dd000
> close(3)                                = 0
> open("/usr/lib/liblttng-ust-libc-wrapper.so.0", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\t\0\0004\0\0\0\364"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=9300, ...}) = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77d8000
> mmap2(NULL, 12104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d5000
> mmap2(0xb77d7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77d7000
> close(3)                                = 0
> open("/usr/lib/liburcu.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d0000
> mmap2(0xb77d4000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77d4000
> close(3)                                = 0
> open("/usr/lib/liburcu-common.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\5\0\0004\0\0\0\34"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=5084, ...}) = 0
> mmap2(NULL, 8032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ce000
> mmap2(0xb77cf000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77cf000
> close(3)                                = 0
> open("/usr/lib/liburcu-qsbr.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\22\0\0004\0\0\0\374"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c9000
> mmap2(0xb77cd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77cd000
> close(3)                                = 0
> open("/usr/lib/liburcu-mb.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c4000
> mmap2(0xb77c8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c8000
> close(3)                                = 0
> open("/usr/lib/liburcu-signal.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\23\0\0004\0\0\0\10"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=18712, ...}) = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77c3000
> mmap2(NULL, 21704, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77bd000
> mmap2(0xb77c2000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c2000
> close(3)                                = 0
> open("/usr/lib/liburcu-cds.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\f\0\0004\0\0\0t"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=26204, ...}) = 0
> mmap2(NULL, 25004, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b6000
> mmap2(0xb77bc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb77bc000
> close(3)                                = 0
> open("/usr/lib/liburcu-bp.so.1", O_RDONLY) = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\24\0\0004\0\0\0\360"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=19968, ...}) = 0
> mmap2(NULL, 23128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b0000
> mmap2(0xb77b5000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77b5000
> close(3)                                = 0
> open("/lib/libdl.so.2", O_RDONLY)       = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \n\0\0004\0\0\0<"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=9668, ...}) = 0
> mmap2(NULL, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ac000
> mmap2(0xb77ae000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77ae000
> close(3)                                = 0
> open("/lib/libuuid.so.1", O_RDONLY)     = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@\r\0\0004\0\0\0\350"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=12872, ...}) = 0
> mmap2(NULL, 15632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77a8000
> mmap2(0xb77ab000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2) = 0xb77ab000
> close(3)                                = 0
> open("/lib/libc.so.6", O_RDONLY)        = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220n\1\0004\0\0\0L"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=1347780, ...}) = 0
> mmap2(NULL, 1358120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb765c000
> mprotect(0xb77a1000, 4096, PROT_NONE)   = 0
> mmap2(0xb77a2000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x145) = 0xb77a2000
> mmap2(0xb77a5000, 10536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb77a5000
> close(3)                                = 0
> open("/lib/librt.so.1", O_RDONLY)       = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\30\0\0004\0\0\0\230"..., 512) = 512
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb765b000
> fstat64(3, {st_mode=S_IFREG|0755, st_size=30616, ...}) = 0
> mmap2(NULL, 33364, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7652000
> mmap2(0xb7659000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7659000
> close(3)                                = 0
> open("/lib/libpthread.so.0", O_RDONLY)  = 3
> read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360J\0\0004\0\0\0\354"..., 512) = 512
> fstat64(3, {st_mode=S_IFREG|0755, st_size=88420, ...}) = 0
> mmap2(NULL, 98828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7639000
> mmap2(0xb764e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0xb764e000
> mmap2(0xb7650000, 4620, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7650000
> close(3)                                = 0
> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7638000
> mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7636000
> set_thread_area({entry_number:-1 -> 6, base_addr:0xb7636d80, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
> mprotect(0xb764e000, 4096, PROT_READ)   = 0
> mprotect(0xb7659000, 4096, PROT_READ)   = 0
> mprotect(0xb77a2000, 8192, PROT_READ)   = 0
> mprotect(0xb77ae000, 4096, PROT_READ)   = 0
> mprotect(0xb785a000, 4096, PROT_READ)   = 0
> munmap(0xb7837000, 17420)               = 0
> set_tid_address(0xb7636de8)             = 3050
> set_robust_list(0xb7636df0, 0xc)        = 0
> futex(0xbfaaf560, FUTEX_WAKE_PRIVATE, 1) = 0
> futex(0xbfaaf560, 0x189 /* FUTEX_??? */, 1, NULL, bfaaf570) = -1 EAGAIN (Resource temporarily unavailable)
> rt_sigaction(SIGRTMIN, {0xb763d4f0, [], SA_SIGINFO}, NULL, 8) = 0
> rt_sigaction(SIGRT_1, {0xb763d9c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
> uname({sys="Linux", node="5410", ...})  = 0
> rt_sigaction(SIGUSR1, {0xb77bec0a, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> futex(0xb77af06c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
> brk(0)                                  = 0x804e000
> brk(0x806f000)                          = 0x806f000
> gettimeofday({1339187216, 880428}, NULL) = 0
> open("/dev/urandom", O_RDONLY|O_LARGEFILE) = 3
> fcntl64(3, F_GETFD)                     = 0
> fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
> getuid32()                              = 0
> gettimeofday({1339187216, 889603}, NULL) = 0
> gettimeofday({1339187216, 894471}, NULL) = 0
> read(3, "\275\0051F\215\373\371\202\377kfb/\362\303N"..., 16) = 16
> clock_gettime(CLOCK_REALTIME, {1339187216, 902329405}) = 0
> getuid32()                              = 0
> geteuid32()                             = 0
> rt_sigprocmask(SIG_SETMASK, ~[RTMIN RT_1], [], 8) = 0
> mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6e36000
> mprotect(0xb6e36000, 4096, PROT_NONE)   = 0
> clone(child_stack=0xb7634d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb7635bd8, {entry_number:6, base_addr:0xb7635b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb7635bd8) = 3051
> mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6636000
> mprotect(0xb6636000, 4096, PROT_NONE)   = 0
> clone(child_stack=0xb6e34d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb6e35bd8, {entry_number:6, base_addr:0xb6e35b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb6e35bd8) = 3052
> rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
> gettimeofday({1339187216, 974869}, NULL) = 0
> futex(0xb7836e30, FUTEX_WAIT_PRIVATE, 0, {2, 927460405}) = -1 ETIMEDOUT (Connection timed out)
> gettid()                                = 3050
> write(2, "libust[3050/3050]: Error: Timed o"..., 107libust[3050/3050]: Error: Timed out waiting for ltt-sessiond (in lttng_ust_init() at lttng-ust-comm.c:912)
> ) = 107
> futex(0xb7836e80, FUTEX_WAIT_PRIVATE, 2, NULL
> _______________________________________________
> lttng-dev mailing list
> lttng-dev@lists.lttng.org
> http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev


--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

^ permalink raw reply	[flat|nested] 16+ messages in thread
* Re: UST Hanging at Tracepoint
       [not found] ` <3F56B905CF2083408E8482044F20428AEF78BC79@ONWVEXCHMB01.ciena.com>
@ 2012-06-12 14:09   ` Mathieu Desnoyers
       [not found]   ` <20120612140925.GB13917@Krystal>
  1 sibling, 0 replies; 16+ messages in thread
From: Mathieu Desnoyers @ 2012-06-12 14:09 UTC (permalink / raw)
  To: Burton, Michael, dgoulet; +Cc: lttng-dev

Hi Michael,

Please note that you should be able to run UST with either (or both) of:

- root lttng-sessiond, for system-wide tracing
- per-user lttng-sessiond, for per-user tracing.

It is entirely normal that one of the two UST threads within the
application can't find an active sessiond if, for instance, you have
just a root (and no per-user) sessiond running.

So although I'm glad that you got things working, I fail to understand
your explanation: it should have worked fine with the root sessiond,
even if no ~/.lttng exists.

David, can you look into this ?

Thanks,

Mathieu

* Burton, Michael (mburton@ciena.com) wrote:
> Mathieu,
> 
> I figured out the problem.  When we start sessiond on our system as root, the /.lttng/ folder is not created, along with its contents.  UST timed out since there was no sock_path.  I was able to get UST working once I got sessiond running.
> 
> Thanks.
> Michael
> 
> ________________________________________
> From: Mathieu Desnoyers [mathieu.desnoyers@efficios.com]
> Sent: 08 June 2012 21:38
> To: Burton, Michael
> Cc: lttng-dev@lists.lttng.org
> Subject: Re: [lttng-dev] UST Hanging at Tracepoint
> 
> * Burton, Michael (mburton@ciena.com) wrote:
> > For further insight, I've attached the strace from the hanging
> > program.  I also upgraded LTTng-UST to 2.0.3 and userspace-rcu to
> > 0.7.3.
> 
> Haven't had time to look at it yet, but could you also provide the
> output of:
> 
> LTTNG_UST_DEBUG=1 youprogname
> 
> (starting your UST instrumented program with LTTNG_UST_DEBUG=1 env. var.
> set)
> 
> Thanks!
> 
> Mathieu
> 
> >
> > Michael
> >
> > From: Burton, Michael [mailto:mburton@ciena.com]
> > Sent: Thursday, June 07, 2012 2:16 PM
> > To: lttng-dev@lists.lttng.org
> > Subject: [lttng-dev] UST Hanging at Tracepoint
> >
> > I am working on getting LTTng 2.0 working in our 2.6.35 kernel.  I have kernel tracing working but I'm having problems with UST.
> >
> > I have a program with a dynamic UST tracepoint.  When I run the program with all UST events enabled (lttng enable-event -a -u) the program hangs on the tracepoint.  The last line from the UST debug is:
> >
> > libust[6184/6185]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)
> >
> > Any insight into why this is happening?  I've attached the UST and lttng-sessiond debug generated by the program.
> >
> > I am running the following:
> > lttng-modules-2.6.32 (found through this mailing list)
> > lttng-tools-2.0.1
> > lttng-ust-2.0.2
> > userspace-rcu-0.7.0
> >
> > Thanks,
> > Michael
> 
> Content-Description: strace_debug.txt
> > execve("/ciena/bin/idp", ["idp", "help"], [/* 11 vars */]) = 0
> > brk(0)                                  = 0x804e000
> > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb783c000
> > access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
> > open("/etc/ld.so.cache", O_RDONLY)      = 3
> > fstat64(3, {st_mode=S_IFREG|0644, st_size=17420, ...}) = 0
> > mmap2(NULL, 17420, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7837000
> > close(3)                                = 0
> > open("/usr/lib/liblttng-ust.so.0", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220A\0\0004\0\0\0$"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=209876, ...}) = 0
> > mmap2(NULL, 212940, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7803000
> > mmap2(0xb7832000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2e) = 0xb7832000
> > close(3)                                = 0
> > open("/usr/lib/liblttng-ust-ctl.so.0", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200-\0\0004\0\0\0\274"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=140020, ...}) = 0
> > mmap2(NULL, 142828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77e0000
> > mmap2(0xb7802000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21) = 0xb7802000
> > close(3)                                = 0
> > open("/usr/lib/liblttng-ust-fork.so.0", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\5\0\0004\0\0\0X"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=3864, ...}) = 0
> > mmap2(NULL, 6820, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77de000
> > mmap2(0xb77df000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77df000
> > close(3)                                = 0
> > open("/usr/lib/liblttng-ust-tracepoint.so.0", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220\v\0\0004\0\0\0008"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=16632, ...}) = 0
> > mmap2(NULL, 19944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d9000
> > mmap2(0xb77dd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xb77dd000
> > close(3)                                = 0
> > open("/usr/lib/liblttng-ust-libc-wrapper.so.0", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\t\0\0004\0\0\0\364"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=9300, ...}) = 0
> > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77d8000
> > mmap2(NULL, 12104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d5000
> > mmap2(0xb77d7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77d7000
> > close(3)                                = 0
> > open("/usr/lib/liburcu.so.1", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d0000
> > mmap2(0xb77d4000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77d4000
> > close(3)                                = 0
> > open("/usr/lib/liburcu-common.so.1", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\5\0\0004\0\0\0\34"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=5084, ...}) = 0
> > mmap2(NULL, 8032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ce000
> > mmap2(0xb77cf000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77cf000
> > close(3)                                = 0
> > open("/usr/lib/liburcu-qsbr.so.1", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\22\0\0004\0\0\0\374"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c9000
> > mmap2(0xb77cd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77cd000
> > close(3)                                = 0
> > open("/usr/lib/liburcu-mb.so.1", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c4000
> > mmap2(0xb77c8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c8000
> > close(3)                                = 0
> > open("/usr/lib/liburcu-signal.so.1", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\23\0\0004\0\0\0\10"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=18712, ...}) = 0
> > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77c3000
> > mmap2(NULL, 21704, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77bd000
> > mmap2(0xb77c2000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c2000
> > close(3)                                = 0
> > open("/usr/lib/liburcu-cds.so.1", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\f\0\0004\0\0\0t"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=26204, ...}) = 0
> > mmap2(NULL, 25004, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b6000
> > mmap2(0xb77bc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb77bc000
> > close(3)                                = 0
> > open("/usr/lib/liburcu-bp.so.1", O_RDONLY) = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\24\0\0004\0\0\0\360"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=19968, ...}) = 0
> > mmap2(NULL, 23128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b0000
> > mmap2(0xb77b5000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77b5000
> > close(3)                                = 0
> > open("/lib/libdl.so.2", O_RDONLY)       = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \n\0\0004\0\0\0<"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=9668, ...}) = 0
> > mmap2(NULL, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ac000
> > mmap2(0xb77ae000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77ae000
> > close(3)                                = 0
> > open("/lib/libuuid.so.1", O_RDONLY)     = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@\r\0\0004\0\0\0\350"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=12872, ...}) = 0
> > mmap2(NULL, 15632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77a8000
> > mmap2(0xb77ab000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2) = 0xb77ab000
> > close(3)                                = 0
> > open("/lib/libc.so.6", O_RDONLY)        = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220n\1\0004\0\0\0L"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=1347780, ...}) = 0
> > mmap2(NULL, 1358120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb765c000
> > mprotect(0xb77a1000, 4096, PROT_NONE)   = 0
> > mmap2(0xb77a2000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x145) = 0xb77a2000
> > mmap2(0xb77a5000, 10536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb77a5000
> > close(3)                                = 0
> > open("/lib/librt.so.1", O_RDONLY)       = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\30\0\0004\0\0\0\230"..., 512) = 512
> > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb765b000
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=30616, ...}) = 0
> > mmap2(NULL, 33364, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7652000
> > mmap2(0xb7659000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7659000
> > close(3)                                = 0
> > open("/lib/libpthread.so.0", O_RDONLY)  = 3
> > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360J\0\0004\0\0\0\354"..., 512) = 512
> > fstat64(3, {st_mode=S_IFREG|0755, st_size=88420, ...}) = 0
> > mmap2(NULL, 98828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7639000
> > mmap2(0xb764e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0xb764e000
> > mmap2(0xb7650000, 4620, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7650000
> > close(3)                                = 0
> > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7638000
> > mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7636000
> > set_thread_area({entry_number:-1 -> 6, base_addr:0xb7636d80, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
> > mprotect(0xb764e000, 4096, PROT_READ)   = 0
> > mprotect(0xb7659000, 4096, PROT_READ)   = 0
> > mprotect(0xb77a2000, 8192, PROT_READ)   = 0
> > mprotect(0xb77ae000, 4096, PROT_READ)   = 0
> > mprotect(0xb785a000, 4096, PROT_READ)   = 0
> > munmap(0xb7837000, 17420)               = 0
> > set_tid_address(0xb7636de8)             = 3050
> > set_robust_list(0xb7636df0, 0xc)        = 0
> > futex(0xbfaaf560, FUTEX_WAKE_PRIVATE, 1) = 0
> > futex(0xbfaaf560, 0x189 /* FUTEX_??? */, 1, NULL, bfaaf570) = -1 EAGAIN (Resource temporarily unavailable)
> > rt_sigaction(SIGRTMIN, {0xb763d4f0, [], SA_SIGINFO}, NULL, 8) = 0
> > rt_sigaction(SIGRT_1, {0xb763d9c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> > getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
> > uname({sys="Linux", node="5410", ...})  = 0
> > rt_sigaction(SIGUSR1, {0xb77bec0a, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > futex(0xb77af06c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
> > brk(0)                                  = 0x804e000
> > brk(0x806f000)                          = 0x806f000
> > gettimeofday({1339187216, 880428}, NULL) = 0
> > open("/dev/urandom", O_RDONLY|O_LARGEFILE) = 3
> > fcntl64(3, F_GETFD)                     = 0
> > fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
> > getuid32()                              = 0
> > gettimeofday({1339187216, 889603}, NULL) = 0
> > gettimeofday({1339187216, 894471}, NULL) = 0
> > read(3, "\275\0051F\215\373\371\202\377kfb/\362\303N"..., 16) = 16
> > clock_gettime(CLOCK_REALTIME, {1339187216, 902329405}) = 0
> > getuid32()                              = 0
> > geteuid32()                             = 0
> > rt_sigprocmask(SIG_SETMASK, ~[RTMIN RT_1], [], 8) = 0
> > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6e36000
> > mprotect(0xb6e36000, 4096, PROT_NONE)   = 0
> > clone(child_stack=0xb7634d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb7635bd8, {entry_number:6, base_addr:0xb7635b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb7635bd8) = 3051
> > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6636000
> > mprotect(0xb6636000, 4096, PROT_NONE)   = 0
> > clone(child_stack=0xb6e34d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb6e35bd8, {entry_number:6, base_addr:0xb6e35b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb6e35bd8) = 3052
> > rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
> > gettimeofday({1339187216, 974869}, NULL) = 0
> > futex(0xb7836e30, FUTEX_WAIT_PRIVATE, 0, {2, 927460405}) = -1 ETIMEDOUT (Connection timed out)
> > gettid()                                = 3050
> > write(2, "libust[3050/3050]: Error: Timed o"..., 107libust[3050/3050]: Error: Timed out waiting for ltt-sessiond (in lttng_ust_init() at lttng-ust-comm.c:912)
> > ) = 107
> > futex(0xb7836e80, FUTEX_WAIT_PRIVATE, 2, NULL
> > _______________________________________________
> > lttng-dev mailing list
> > lttng-dev@lists.lttng.org
> > http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
> 
> 
> --
> Mathieu Desnoyers
> Operating System Efficiency R&D Consultant
> EfficiOS Inc.
> http://www.efficios.com

-- 
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

^ permalink raw reply	[flat|nested] 16+ messages in thread
* Re: UST Hanging at Tracepoint
       [not found]   ` <20120612140925.GB13917@Krystal>
@ 2012-06-12 14:13     ` Mathieu Desnoyers
       [not found]     ` <20120612141344.GC13917@Krystal>
  2012-06-12 15:51     ` David Goulet
  2 siblings, 0 replies; 16+ messages in thread
From: Mathieu Desnoyers @ 2012-06-12 14:13 UTC (permalink / raw)
  To: Burton, Michael, dgoulet; +Cc: lttng-dev

* Mathieu Desnoyers (mathieu.desnoyers@efficios.com) wrote:
> Hi Michael,
> 
> Please note that you should be able to run UST with either (or both) of:
> 
> - root lttng-sessiond, for system-wide tracing
> - per-user lttng-sessiond, for per-user tracing.
> 
> It is entirely normal that one of the two UST threads within the
> application can't find an active sessiond if, for instance, you have
> just a root (and no per-user) sessiond running.
> 
> So although I'm glad that you got things working, I fail to understand
> your explanation: it should have worked fine with the root sessiond,
> even if no ~/.lttng exists.
> 
> David, can you look into this ?

hrm, further insight: is your program _really_ hang ? Try to make sure
you put a fprintf(stderr, ".....") at the beginning of your main().
Looking once more at your detailed UST output indicates that the
per-user UST thread tells you that it cannot see any running sessiond.
So yes, this thread blocks, but the thread talking to the root sessiond
can very well be working fine, and your application might also be
running fine. This should be double-checked.

Thanks,

Mathieu

> 
> Thanks,
> 
> Mathieu
> 
> * Burton, Michael (mburton@ciena.com) wrote:
> > Mathieu,
> > 
> > I figured out the problem.  When we start sessiond on our system as root, the /.lttng/ folder is not created, along with its contents.  UST timed out since there was no sock_path.  I was able to get UST working once I got sessiond running.
> > 
> > Thanks.
> > Michael
> > 
> > ________________________________________
> > From: Mathieu Desnoyers [mathieu.desnoyers@efficios.com]
> > Sent: 08 June 2012 21:38
> > To: Burton, Michael
> > Cc: lttng-dev@lists.lttng.org
> > Subject: Re: [lttng-dev] UST Hanging at Tracepoint
> > 
> > * Burton, Michael (mburton@ciena.com) wrote:
> > > For further insight, I've attached the strace from the hanging
> > > program.  I also upgraded LTTng-UST to 2.0.3 and userspace-rcu to
> > > 0.7.3.
> > 
> > Haven't had time to look at it yet, but could you also provide the
> > output of:
> > 
> > LTTNG_UST_DEBUG=1 youprogname
> > 
> > (starting your UST instrumented program with LTTNG_UST_DEBUG=1 env. var.
> > set)
> > 
> > Thanks!
> > 
> > Mathieu
> > 
> > >
> > > Michael
> > >
> > > From: Burton, Michael [mailto:mburton@ciena.com]
> > > Sent: Thursday, June 07, 2012 2:16 PM
> > > To: lttng-dev@lists.lttng.org
> > > Subject: [lttng-dev] UST Hanging at Tracepoint
> > >
> > > I am working on getting LTTng 2.0 working in our 2.6.35 kernel.  I have kernel tracing working but I'm having problems with UST.
> > >
> > > I have a program with a dynamic UST tracepoint.  When I run the program with all UST events enabled (lttng enable-event -a -u) the program hangs on the tracepoint.  The last line from the UST debug is:
> > >
> > > libust[6184/6185]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)
> > >
> > > Any insight into why this is happening?  I've attached the UST and lttng-sessiond debug generated by the program.
> > >
> > > I am running the following:
> > > lttng-modules-2.6.32 (found through this mailing list)
> > > lttng-tools-2.0.1
> > > lttng-ust-2.0.2
> > > userspace-rcu-0.7.0
> > >
> > > Thanks,
> > > Michael
> > 
> > Content-Description: strace_debug.txt
> > > execve("/ciena/bin/idp", ["idp", "help"], [/* 11 vars */]) = 0
> > > brk(0)                                  = 0x804e000
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb783c000
> > > access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
> > > open("/etc/ld.so.cache", O_RDONLY)      = 3
> > > fstat64(3, {st_mode=S_IFREG|0644, st_size=17420, ...}) = 0
> > > mmap2(NULL, 17420, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7837000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220A\0\0004\0\0\0$"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=209876, ...}) = 0
> > > mmap2(NULL, 212940, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7803000
> > > mmap2(0xb7832000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2e) = 0xb7832000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust-ctl.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200-\0\0004\0\0\0\274"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=140020, ...}) = 0
> > > mmap2(NULL, 142828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77e0000
> > > mmap2(0xb7802000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21) = 0xb7802000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust-fork.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\5\0\0004\0\0\0X"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=3864, ...}) = 0
> > > mmap2(NULL, 6820, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77de000
> > > mmap2(0xb77df000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77df000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust-tracepoint.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220\v\0\0004\0\0\0008"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=16632, ...}) = 0
> > > mmap2(NULL, 19944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d9000
> > > mmap2(0xb77dd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xb77dd000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust-libc-wrapper.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\t\0\0004\0\0\0\364"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=9300, ...}) = 0
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77d8000
> > > mmap2(NULL, 12104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d5000
> > > mmap2(0xb77d7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77d7000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d0000
> > > mmap2(0xb77d4000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77d4000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-common.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\5\0\0004\0\0\0\34"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=5084, ...}) = 0
> > > mmap2(NULL, 8032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ce000
> > > mmap2(0xb77cf000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77cf000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-qsbr.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\22\0\0004\0\0\0\374"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c9000
> > > mmap2(0xb77cd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77cd000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-mb.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c4000
> > > mmap2(0xb77c8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c8000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-signal.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\23\0\0004\0\0\0\10"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18712, ...}) = 0
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77c3000
> > > mmap2(NULL, 21704, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77bd000
> > > mmap2(0xb77c2000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c2000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-cds.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\f\0\0004\0\0\0t"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=26204, ...}) = 0
> > > mmap2(NULL, 25004, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b6000
> > > mmap2(0xb77bc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb77bc000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-bp.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\24\0\0004\0\0\0\360"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=19968, ...}) = 0
> > > mmap2(NULL, 23128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b0000
> > > mmap2(0xb77b5000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77b5000
> > > close(3)                                = 0
> > > open("/lib/libdl.so.2", O_RDONLY)       = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \n\0\0004\0\0\0<"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=9668, ...}) = 0
> > > mmap2(NULL, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ac000
> > > mmap2(0xb77ae000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77ae000
> > > close(3)                                = 0
> > > open("/lib/libuuid.so.1", O_RDONLY)     = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@\r\0\0004\0\0\0\350"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=12872, ...}) = 0
> > > mmap2(NULL, 15632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77a8000
> > > mmap2(0xb77ab000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2) = 0xb77ab000
> > > close(3)                                = 0
> > > open("/lib/libc.so.6", O_RDONLY)        = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220n\1\0004\0\0\0L"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=1347780, ...}) = 0
> > > mmap2(NULL, 1358120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb765c000
> > > mprotect(0xb77a1000, 4096, PROT_NONE)   = 0
> > > mmap2(0xb77a2000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x145) = 0xb77a2000
> > > mmap2(0xb77a5000, 10536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb77a5000
> > > close(3)                                = 0
> > > open("/lib/librt.so.1", O_RDONLY)       = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\30\0\0004\0\0\0\230"..., 512) = 512
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb765b000
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=30616, ...}) = 0
> > > mmap2(NULL, 33364, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7652000
> > > mmap2(0xb7659000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7659000
> > > close(3)                                = 0
> > > open("/lib/libpthread.so.0", O_RDONLY)  = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360J\0\0004\0\0\0\354"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=88420, ...}) = 0
> > > mmap2(NULL, 98828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7639000
> > > mmap2(0xb764e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0xb764e000
> > > mmap2(0xb7650000, 4620, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7650000
> > > close(3)                                = 0
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7638000
> > > mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7636000
> > > set_thread_area({entry_number:-1 -> 6, base_addr:0xb7636d80, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
> > > mprotect(0xb764e000, 4096, PROT_READ)   = 0
> > > mprotect(0xb7659000, 4096, PROT_READ)   = 0
> > > mprotect(0xb77a2000, 8192, PROT_READ)   = 0
> > > mprotect(0xb77ae000, 4096, PROT_READ)   = 0
> > > mprotect(0xb785a000, 4096, PROT_READ)   = 0
> > > munmap(0xb7837000, 17420)               = 0
> > > set_tid_address(0xb7636de8)             = 3050
> > > set_robust_list(0xb7636df0, 0xc)        = 0
> > > futex(0xbfaaf560, FUTEX_WAKE_PRIVATE, 1) = 0
> > > futex(0xbfaaf560, 0x189 /* FUTEX_??? */, 1, NULL, bfaaf570) = -1 EAGAIN (Resource temporarily unavailable)
> > > rt_sigaction(SIGRTMIN, {0xb763d4f0, [], SA_SIGINFO}, NULL, 8) = 0
> > > rt_sigaction(SIGRT_1, {0xb763d9c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > > rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> > > getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
> > > uname({sys="Linux", node="5410", ...})  = 0
> > > rt_sigaction(SIGUSR1, {0xb77bec0a, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > > futex(0xb77af06c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
> > > brk(0)                                  = 0x804e000
> > > brk(0x806f000)                          = 0x806f000
> > > gettimeofday({1339187216, 880428}, NULL) = 0
> > > open("/dev/urandom", O_RDONLY|O_LARGEFILE) = 3
> > > fcntl64(3, F_GETFD)                     = 0
> > > fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
> > > getuid32()                              = 0
> > > gettimeofday({1339187216, 889603}, NULL) = 0
> > > gettimeofday({1339187216, 894471}, NULL) = 0
> > > read(3, "\275\0051F\215\373\371\202\377kfb/\362\303N"..., 16) = 16
> > > clock_gettime(CLOCK_REALTIME, {1339187216, 902329405}) = 0
> > > getuid32()                              = 0
> > > geteuid32()                             = 0
> > > rt_sigprocmask(SIG_SETMASK, ~[RTMIN RT_1], [], 8) = 0
> > > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6e36000
> > > mprotect(0xb6e36000, 4096, PROT_NONE)   = 0
> > > clone(child_stack=0xb7634d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb7635bd8, {entry_number:6, base_addr:0xb7635b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb7635bd8) = 3051
> > > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6636000
> > > mprotect(0xb6636000, 4096, PROT_NONE)   = 0
> > > clone(child_stack=0xb6e34d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb6e35bd8, {entry_number:6, base_addr:0xb6e35b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb6e35bd8) = 3052
> > > rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
> > > gettimeofday({1339187216, 974869}, NULL) = 0
> > > futex(0xb7836e30, FUTEX_WAIT_PRIVATE, 0, {2, 927460405}) = -1 ETIMEDOUT (Connection timed out)
> > > gettid()                                = 3050
> > > write(2, "libust[3050/3050]: Error: Timed o"..., 107libust[3050/3050]: Error: Timed out waiting for ltt-sessiond (in lttng_ust_init() at lttng-ust-comm.c:912)
> > > ) = 107
> > > futex(0xb7836e80, FUTEX_WAIT_PRIVATE, 2, NULL
> > > _______________________________________________
> > > lttng-dev mailing list
> > > lttng-dev@lists.lttng.org
> > > http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
> > 
> > 
> > --
> > Mathieu Desnoyers
> > Operating System Efficiency R&D Consultant
> > EfficiOS Inc.
> > http://www.efficios.com
> 
> -- 
> Mathieu Desnoyers
> Operating System Efficiency R&D Consultant
> EfficiOS Inc.
> http://www.efficios.com
> 
> _______________________________________________
> lttng-dev mailing list
> lttng-dev@lists.lttng.org
> http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

-- 
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

^ permalink raw reply	[flat|nested] 16+ messages in thread
* Re: UST Hanging at Tracepoint
       [not found]     ` <20120612141344.GC13917@Krystal>
@ 2012-06-12 15:44       ` Burton, Michael
       [not found]       ` <3F56B905CF2083408E8482044F20428AEF7A0AFE@ONWVEXCHMB01.ciena.com>
  1 sibling, 0 replies; 16+ messages in thread
From: Burton, Michael @ 2012-06-12 15:44 UTC (permalink / raw)
  To: Mathieu Desnoyers, dgoulet; +Cc: lttng-dev

I put a fprintf at the beginning of main() and it never prints.

-----Original Message-----
From: Mathieu Desnoyers [mailto:mathieu.desnoyers@efficios.com]
Sent: Tuesday, June 12, 2012 10:14 AM
To: Burton, Michael; dgoulet@efficios.com
Cc: lttng-dev@lists.lttng.org
Subject: Re: [lttng-dev] UST Hanging at Tracepoint

* Mathieu Desnoyers (mathieu.desnoyers@efficios.com) wrote:
> Hi Michael,
>
> Please note that you should be able to run UST with either (or both) of:
>
> - root lttng-sessiond, for system-wide tracing
> - per-user lttng-sessiond, for per-user tracing.
>
> It is entirely normal that one of the two UST threads within the
> application can't find an active sessiond if, for instance, you have
> just a root (and no per-user) sessiond running.
>
> So although I'm glad that you got things working, I fail to understand
> your explanation: it should have worked fine with the root sessiond,
> even if no ~/.lttng exists.
>
> David, can you look into this ?

hrm, further insight: is your program _really_ hang ? Try to make sure
you put a fprintf(stderr, ".....") at the beginning of your main().
Looking once more at your detailed UST output indicates that the
per-user UST thread tells you that it cannot see any running sessiond.
So yes, this thread blocks, but the thread talking to the root sessiond
can very well be working fine, and your application might also be
running fine. This should be double-checked.

Thanks,

Mathieu

>
> Thanks,
>
> Mathieu
>
> * Burton, Michael (mburton@ciena.com) wrote:
> > Mathieu,
> >
> > I figured out the problem.  When we start sessiond on our system as root, the /.lttng/ folder is not created, along with its contents.  UST timed out since there was no sock_path.  I was able to get UST working once I got sessiond running.
> >
> > Thanks.
> > Michael
> >
> > ________________________________________
> > From: Mathieu Desnoyers [mathieu.desnoyers@efficios.com]
> > Sent: 08 June 2012 21:38
> > To: Burton, Michael
> > Cc: lttng-dev@lists.lttng.org
> > Subject: Re: [lttng-dev] UST Hanging at Tracepoint
> >
> > * Burton, Michael (mburton@ciena.com) wrote:
> > > For further insight, I've attached the strace from the hanging
> > > program.  I also upgraded LTTng-UST to 2.0.3 and userspace-rcu to
> > > 0.7.3.
> >
> > Haven't had time to look at it yet, but could you also provide the
> > output of:
> >
> > LTTNG_UST_DEBUG=1 youprogname
> >
> > (starting your UST instrumented program with LTTNG_UST_DEBUG=1 env. var.
> > set)
> >
> > Thanks!
> >
> > Mathieu
> >
> > >
> > > Michael
> > >
> > > From: Burton, Michael [mailto:mburton@ciena.com]
> > > Sent: Thursday, June 07, 2012 2:16 PM
> > > To: lttng-dev@lists.lttng.org
> > > Subject: [lttng-dev] UST Hanging at Tracepoint
> > >
> > > I am working on getting LTTng 2.0 working in our 2.6.35 kernel.  I have kernel tracing working but I'm having problems with UST.
> > >
> > > I have a program with a dynamic UST tracepoint.  When I run the program with all UST events enabled (lttng enable-event -a -u) the program hangs on the tracepoint.  The last line from the UST debug is:
> > >
> > > libust[6184/6185]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)
> > >
> > > Any insight into why this is happening?  I've attached the UST and lttng-sessiond debug generated by the program.
> > >
> > > I am running the following:
> > > lttng-modules-2.6.32 (found through this mailing list)
> > > lttng-tools-2.0.1
> > > lttng-ust-2.0.2
> > > userspace-rcu-0.7.0
> > >
> > > Thanks,
> > > Michael
> >
> > Content-Description: strace_debug.txt
> > > execve("/ciena/bin/idp", ["idp", "help"], [/* 11 vars */]) = 0
> > > brk(0)                                  = 0x804e000
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb783c000
> > > access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
> > > open("/etc/ld.so.cache", O_RDONLY)      = 3
> > > fstat64(3, {st_mode=S_IFREG|0644, st_size=17420, ...}) = 0
> > > mmap2(NULL, 17420, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7837000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220A\0\0004\0\0\0$"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=209876, ...}) = 0
> > > mmap2(NULL, 212940, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7803000
> > > mmap2(0xb7832000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2e) = 0xb7832000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust-ctl.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200-\0\0004\0\0\0\274"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=140020, ...}) = 0
> > > mmap2(NULL, 142828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77e0000
> > > mmap2(0xb7802000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21) = 0xb7802000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust-fork.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\5\0\0004\0\0\0X"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=3864, ...}) = 0
> > > mmap2(NULL, 6820, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77de000
> > > mmap2(0xb77df000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77df000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust-tracepoint.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220\v\0\0004\0\0\0008"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=16632, ...}) = 0
> > > mmap2(NULL, 19944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d9000
> > > mmap2(0xb77dd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xb77dd000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust-libc-wrapper.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\t\0\0004\0\0\0\364"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=9300, ...}) = 0
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77d8000
> > > mmap2(NULL, 12104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d5000
> > > mmap2(0xb77d7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77d7000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d0000
> > > mmap2(0xb77d4000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77d4000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-common.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\5\0\0004\0\0\0\34"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=5084, ...}) = 0
> > > mmap2(NULL, 8032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ce000
> > > mmap2(0xb77cf000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77cf000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-qsbr.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\22\0\0004\0\0\0\374"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c9000
> > > mmap2(0xb77cd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77cd000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-mb.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c4000
> > > mmap2(0xb77c8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c8000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-signal.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\23\0\0004\0\0\0\10"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18712, ...}) = 0
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77c3000
> > > mmap2(NULL, 21704, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77bd000
> > > mmap2(0xb77c2000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c2000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-cds.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\f\0\0004\0\0\0t"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=26204, ...}) = 0
> > > mmap2(NULL, 25004, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b6000
> > > mmap2(0xb77bc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb77bc000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-bp.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\24\0\0004\0\0\0\360"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=19968, ...}) = 0
> > > mmap2(NULL, 23128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b0000
> > > mmap2(0xb77b5000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77b5000
> > > close(3)                                = 0
> > > open("/lib/libdl.so.2", O_RDONLY)       = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \n\0\0004\0\0\0<"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=9668, ...}) = 0
> > > mmap2(NULL, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ac000
> > > mmap2(0xb77ae000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77ae000
> > > close(3)                                = 0
> > > open("/lib/libuuid.so.1", O_RDONLY)     = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@\r\0\0004\0\0\0\350"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=12872, ...}) = 0
> > > mmap2(NULL, 15632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77a8000
> > > mmap2(0xb77ab000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2) = 0xb77ab000
> > > close(3)                                = 0
> > > open("/lib/libc.so.6", O_RDONLY)        = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220n\1\0004\0\0\0L"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=1347780, ...}) = 0
> > > mmap2(NULL, 1358120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb765c000
> > > mprotect(0xb77a1000, 4096, PROT_NONE)   = 0
> > > mmap2(0xb77a2000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x145) = 0xb77a2000
> > > mmap2(0xb77a5000, 10536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb77a5000
> > > close(3)                                = 0
> > > open("/lib/librt.so.1", O_RDONLY)       = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\30\0\0004\0\0\0\230"..., 512) = 512
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb765b000
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=30616, ...}) = 0
> > > mmap2(NULL, 33364, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7652000
> > > mmap2(0xb7659000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7659000
> > > close(3)                                = 0
> > > open("/lib/libpthread.so.0", O_RDONLY)  = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360J\0\0004\0\0\0\354"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=88420, ...}) = 0
> > > mmap2(NULL, 98828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7639000
> > > mmap2(0xb764e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0xb764e000
> > > mmap2(0xb7650000, 4620, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7650000
> > > close(3)                                = 0
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7638000
> > > mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7636000
> > > set_thread_area({entry_number:-1 -> 6, base_addr:0xb7636d80, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
> > > mprotect(0xb764e000, 4096, PROT_READ)   = 0
> > > mprotect(0xb7659000, 4096, PROT_READ)   = 0
> > > mprotect(0xb77a2000, 8192, PROT_READ)   = 0
> > > mprotect(0xb77ae000, 4096, PROT_READ)   = 0
> > > mprotect(0xb785a000, 4096, PROT_READ)   = 0
> > > munmap(0xb7837000, 17420)               = 0
> > > set_tid_address(0xb7636de8)             = 3050
> > > set_robust_list(0xb7636df0, 0xc)        = 0
> > > futex(0xbfaaf560, FUTEX_WAKE_PRIVATE, 1) = 0
> > > futex(0xbfaaf560, 0x189 /* FUTEX_??? */, 1, NULL, bfaaf570) = -1 EAGAIN (Resource temporarily unavailable)
> > > rt_sigaction(SIGRTMIN, {0xb763d4f0, [], SA_SIGINFO}, NULL, 8) = 0
> > > rt_sigaction(SIGRT_1, {0xb763d9c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > > rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> > > getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
> > > uname({sys="Linux", node="5410", ...})  = 0
> > > rt_sigaction(SIGUSR1, {0xb77bec0a, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > > futex(0xb77af06c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
> > > brk(0)                                  = 0x804e000
> > > brk(0x806f000)                          = 0x806f000
> > > gettimeofday({1339187216, 880428}, NULL) = 0
> > > open("/dev/urandom", O_RDONLY|O_LARGEFILE) = 3
> > > fcntl64(3, F_GETFD)                     = 0
> > > fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
> > > getuid32()                              = 0
> > > gettimeofday({1339187216, 889603}, NULL) = 0
> > > gettimeofday({1339187216, 894471}, NULL) = 0
> > > read(3, "\275\0051F\215\373\371\202\377kfb/\362\303N"..., 16) = 16
> > > clock_gettime(CLOCK_REALTIME, {1339187216, 902329405}) = 0
> > > getuid32()                              = 0
> > > geteuid32()                             = 0
> > > rt_sigprocmask(SIG_SETMASK, ~[RTMIN RT_1], [], 8) = 0
> > > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6e36000
> > > mprotect(0xb6e36000, 4096, PROT_NONE)   = 0
> > > clone(child_stack=0xb7634d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb7635bd8, {entry_number:6, base_addr:0xb7635b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb7635bd8) = 3051
> > > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6636000
> > > mprotect(0xb6636000, 4096, PROT_NONE)   = 0
> > > clone(child_stack=0xb6e34d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb6e35bd8, {entry_number:6, base_addr:0xb6e35b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb6e35bd8) = 3052
> > > rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
> > > gettimeofday({1339187216, 974869}, NULL) = 0
> > > futex(0xb7836e30, FUTEX_WAIT_PRIVATE, 0, {2, 927460405}) = -1 ETIMEDOUT (Connection timed out)
> > > gettid()                                = 3050
> > > write(2, "libust[3050/3050]: Error: Timed o"..., 107libust[3050/3050]: Error: Timed out waiting for ltt-sessiond (in lttng_ust_init() at lttng-ust-comm.c:912)
> > > ) = 107
> > > futex(0xb7836e80, FUTEX_WAIT_PRIVATE, 2, NULL
> > > _______________________________________________
> > > lttng-dev mailing list
> > > lttng-dev@lists.lttng.org
> > > http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
> >
> >
> > --
> > Mathieu Desnoyers
> > Operating System Efficiency R&D Consultant
> > EfficiOS Inc.
> > http://www.efficios.com
>
> --
> Mathieu Desnoyers
> Operating System Efficiency R&D Consultant
> EfficiOS Inc.
> http://www.efficios.com
>
> _______________________________________________
> lttng-dev mailing list
> lttng-dev@lists.lttng.org
> http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

^ permalink raw reply	[flat|nested] 16+ messages in thread
* Re: UST Hanging at Tracepoint
       [not found]   ` <20120612140925.GB13917@Krystal>
  2012-06-12 14:13     ` Mathieu Desnoyers
       [not found]     ` <20120612141344.GC13917@Krystal>
@ 2012-06-12 15:51     ` David Goulet
  2 siblings, 0 replies; 16+ messages in thread
From: David Goulet @ 2012-06-12 15:51 UTC (permalink / raw)
  To: Mathieu Desnoyers; +Cc: lttng-dev

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

Hey!

On 12/06/12 10:09 AM, Mathieu Desnoyers wrote:
> Hi Michael,
> 
> Please note that you should be able to run UST with either (or both) of:
> 
> - root lttng-sessiond, for system-wide tracing - per-user lttng-sessiond,
> for per-user tracing.
> 
> It is entirely normal that one of the two UST threads within the 
> application can't find an active sessiond if, for instance, you have just a
> root (and no per-user) sessiond running.
> 
> So although I'm glad that you got things working, I fail to understand your
> explanation: it should have worked fine with the root sessiond, even if no
> ~/.lttng exists.

A root session daemon does NOT create a ".lttng" directory. For system wide,
we use /var/run/lttng/*.

However, UST tries the $HOME/.lttng/ and /var/run/lttng for the global session
daemon.

- From the strace output, the timeout occurs on the FUTEX_WAIT and not the
socket... However, there might be a missing thread in that output I think...

futex(0xb7836e30, FUTEX_WAIT_PRIVATE, 0, {2, 927460405}) = -1 ETIMEDOUT
(Connection timed out)

David

> 
> David, can you look into this ?
> 
> Thanks,
> 
> Mathieu
> 
> * Burton, Michael (mburton@ciena.com) wrote:
>> Mathieu,
>> 
>> I figured out the problem.  When we start sessiond on our system as root,
>> the /.lttng/ folder is not created, along with its contents.  UST timed
>> out since there was no sock_path.  I was able to get UST working once I
>> got sessiond running.
>> 
>> Thanks. Michael
>> 
>> ________________________________________ From: Mathieu Desnoyers
>> [mathieu.desnoyers@efficios.com] Sent: 08 June 2012 21:38 To: Burton,
>> Michael Cc: lttng-dev@lists.lttng.org Subject: Re: [lttng-dev] UST
>> Hanging at Tracepoint
>> 
>> * Burton, Michael (mburton@ciena.com) wrote:
>>> For further insight, I've attached the strace from the hanging program.
>>> I also upgraded LTTng-UST to 2.0.3 and userspace-rcu to 0.7.3.
>> 
>> Haven't had time to look at it yet, but could you also provide the output
>> of:
>> 
>> LTTNG_UST_DEBUG=1 youprogname
>> 
>> (starting your UST instrumented program with LTTNG_UST_DEBUG=1 env. var. 
>> set)
>> 
>> Thanks!
>> 
>> Mathieu
>> 
>>> 
>>> Michael
>>> 
>>> From: Burton, Michael [mailto:mburton@ciena.com] Sent: Thursday, June
>>> 07, 2012 2:16 PM To: lttng-dev@lists.lttng.org Subject: [lttng-dev] UST
>>> Hanging at Tracepoint
>>> 
>>> I am working on getting LTTng 2.0 working in our 2.6.35 kernel.  I have
>>> kernel tracing working but I'm having problems with UST.
>>> 
>>> I have a program with a dynamic UST tracepoint.  When I run the program
>>> with all UST events enabled (lttng enable-event -a -u) the program
>>> hangs on the tracepoint.  The last line from the UST debug is:
>>> 
>>> libust[6184/6185]: Info: sessiond not accepting connections to local
>>> apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)
>>> 
>>> Any insight into why this is happening?  I've attached the UST and
>>> lttng-sessiond debug generated by the program.
>>> 
>>> I am running the following: lttng-modules-2.6.32 (found through this
>>> mailing list) lttng-tools-2.0.1 lttng-ust-2.0.2 userspace-rcu-0.7.0
>>> 
>>> Thanks, Michael
>> 
>> Content-Description: strace_debug.txt
>>> execve("/ciena/bin/idp", ["idp", "help"], [/* 11 vars */]) = 0 brk(0)
>>> = 0x804e000 mmap2(NULL, 4096, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb783c000 
>>> access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or
>>> directory) open("/etc/ld.so.cache", O_RDONLY)      = 3 fstat64(3,
>>> {st_mode=S_IFREG|0644, st_size=17420, ...}) = 0 mmap2(NULL, 17420,
>>> PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7837000 close(3)
>>> = 0 open("/usr/lib/liblttng-ust.so.0", O_RDONLY) = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220A\0\0004\0\0\0$"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=209876, ...}) = 0 
>>> mmap2(NULL, 212940, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
>>> 0) = 0xb7803000 mmap2(0xb7832000, 20480, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2e) = 0xb7832000 close(3)
>>> = 0 open("/usr/lib/liblttng-ust-ctl.so.0", O_RDONLY) = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200-\0\0004\0\0\0\274"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=140020, ...}) = 0 
>>> mmap2(NULL, 142828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
>>> 0) = 0xb77e0000 mmap2(0xb7802000, 4096, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21) = 0xb7802000 close(3)
>>> = 0 open("/usr/lib/liblttng-ust-fork.so.0", O_RDONLY) = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\5\0\0004\0\0\0X"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=3864, ...}) = 0 
>>> mmap2(NULL, 6820, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
>>> = 0xb77de000 mmap2(0xb77df000, 4096, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77df000 close(3)
>>> = 0 open("/usr/lib/liblttng-ust-tracepoint.so.0", O_RDONLY) = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220\v\0\0004\0\0\0008"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=16632, ...}) = 0 
>>> mmap2(NULL, 19944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
>>> 0) = 0xb77d9000 mmap2(0xb77dd000, 4096, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xb77dd000 close(3)
>>> = 0 open("/usr/lib/liblttng-ust-libc-wrapper.so.0", O_RDONLY) = 3 
>>> read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\t\0\0004\0\0\0\364"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=9300, ...}) = 0 
>>> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
>>> 0) = 0xb77d8000 mmap2(NULL, 12104, PROT_READ|PROT_EXEC,
>>> MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d5000 mmap2(0xb77d7000, 4096,
>>> PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) =
>>> 0xb77d7000 close(3)                                = 0 
>>> open("/usr/lib/liburcu.so.1", O_RDONLY) = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0 
>>> mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
>>> 0) = 0xb77d0000 mmap2(0xb77d4000, 4096, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77d4000 close(3)
>>> = 0 open("/usr/lib/liburcu-common.so.1", O_RDONLY) = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\5\0\0004\0\0\0\34"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=5084, ...}) = 0 
>>> mmap2(NULL, 8032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0)
>>> = 0xb77ce000 mmap2(0xb77cf000, 4096, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77cf000 close(3)
>>> = 0 open("/usr/lib/liburcu-qsbr.so.1", O_RDONLY) = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\22\0\0004\0\0\0\374"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0 
>>> mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
>>> 0) = 0xb77c9000 mmap2(0xb77cd000, 4096, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77cd000 close(3)
>>> = 0 open("/usr/lib/liburcu-mb.so.1", O_RDONLY) = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0 
>>> mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
>>> 0) = 0xb77c4000 mmap2(0xb77c8000, 4096, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c8000 close(3)
>>> = 0 open("/usr/lib/liburcu-signal.so.1", O_RDONLY) = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\23\0\0004\0\0\0\10"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=18712, ...}) = 0 
>>> mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
>>> 0) = 0xb77c3000 mmap2(NULL, 21704, PROT_READ|PROT_EXEC,
>>> MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77bd000 mmap2(0xb77c2000, 4096,
>>> PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) =
>>> 0xb77c2000 close(3)                                = 0 
>>> open("/usr/lib/liburcu-cds.so.1", O_RDONLY) = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\f\0\0004\0\0\0t"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=26204, ...}) = 0 
>>> mmap2(NULL, 25004, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
>>> 0) = 0xb77b6000 mmap2(0xb77bc000, 4096, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb77bc000 close(3)
>>> = 0 open("/usr/lib/liburcu-bp.so.1", O_RDONLY) = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\24\0\0004\0\0\0\360"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=19968, ...}) = 0 
>>> mmap2(NULL, 23128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
>>> 0) = 0xb77b0000 mmap2(0xb77b5000, 4096, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77b5000 close(3)
>>> = 0 open("/lib/libdl.so.2", O_RDONLY)       = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \n\0\0004\0\0\0<"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=9668, ...}) = 0 
>>> mmap2(NULL, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
>>> 0) = 0xb77ac000 mmap2(0xb77ae000, 8192, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77ae000 close(3)
>>> = 0 open("/lib/libuuid.so.1", O_RDONLY)     = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@\r\0\0004\0\0\0\350"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=12872, ...}) = 0 
>>> mmap2(NULL, 15632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
>>> 0) = 0xb77a8000 mmap2(0xb77ab000, 4096, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2) = 0xb77ab000 close(3)
>>> = 0 open("/lib/libc.so.6", O_RDONLY)        = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220n\1\0004\0\0\0L"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=1347780, ...}) =
>>> 0 mmap2(NULL, 1358120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE,
>>> 3, 0) = 0xb765c000 mprotect(0xb77a1000, 4096, PROT_NONE)   = 0 
>>> mmap2(0xb77a2000, 12288, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x145) = 0xb77a2000 
>>> mmap2(0xb77a5000, 10536, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb77a5000 close(3)
>>> = 0 open("/lib/librt.so.1", O_RDONLY)       = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\30\0\0004\0\0\0\230"...,
>>> 512) = 512 mmap2(NULL, 4096, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb765b000 fstat64(3,
>>> {st_mode=S_IFREG|0755, st_size=30616, ...}) = 0 mmap2(NULL, 33364,
>>> PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7652000 
>>> mmap2(0xb7659000, 8192, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7659000 close(3)
>>> = 0 open("/lib/libpthread.so.0", O_RDONLY)  = 3 read(3,
>>> "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360J\0\0004\0\0\0\354"...,
>>> 512) = 512 fstat64(3, {st_mode=S_IFREG|0755, st_size=88420, ...}) = 0 
>>> mmap2(NULL, 98828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
>>> 0) = 0xb7639000 mmap2(0xb764e000, 8192, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0xb764e000 
>>> mmap2(0xb7650000, 4620, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7650000 close(3)
>>> = 0 mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
>>> -1, 0) = 0xb7638000 mmap2(NULL, 8192, PROT_READ|PROT_WRITE,
>>> MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7636000 
>>> set_thread_area({entry_number:-1 -> 6, base_addr:0xb7636d80,
>>> limit:1048575, seg_32bit:1, contents:0, read_exec_only:0,
>>> limit_in_pages:1, seg_not_present:0, useable:1}) = 0 
>>> mprotect(0xb764e000, 4096, PROT_READ)   = 0 mprotect(0xb7659000, 4096,
>>> PROT_READ)   = 0 mprotect(0xb77a2000, 8192, PROT_READ)   = 0 
>>> mprotect(0xb77ae000, 4096, PROT_READ)   = 0 mprotect(0xb785a000, 4096,
>>> PROT_READ)   = 0 munmap(0xb7837000, 17420)               = 0 
>>> set_tid_address(0xb7636de8)             = 3050 
>>> set_robust_list(0xb7636df0, 0xc)        = 0 futex(0xbfaaf560,
>>> FUTEX_WAKE_PRIVATE, 1) = 0 futex(0xbfaaf560, 0x189 /* FUTEX_??? */, 1,
>>> NULL, bfaaf570) = -1 EAGAIN (Resource temporarily unavailable) 
>>> rt_sigaction(SIGRTMIN, {0xb763d4f0, [], SA_SIGINFO}, NULL, 8) = 0 
>>> rt_sigaction(SIGRT_1, {0xb763d9c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8)
>>> = 0 rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0 
>>> getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) =
>>> 0 uname({sys="Linux", node="5410", ...})  = 0 rt_sigaction(SIGUSR1,
>>> {0xb77bec0a, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0 futex(0xb77af06c,
>>> FUTEX_WAKE_PRIVATE, 2147483647) = 0 brk(0)
>>> = 0x804e000 brk(0x806f000)                          = 0x806f000 
>>> gettimeofday({1339187216, 880428}, NULL) = 0 open("/dev/urandom",
>>> O_RDONLY|O_LARGEFILE) = 3 fcntl64(3, F_GETFD)                     = 0 
>>> fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0 getuid32()
>>> = 0 gettimeofday({1339187216, 889603}, NULL) = 0 
>>> gettimeofday({1339187216, 894471}, NULL) = 0 read(3,
>>> "\275\0051F\215\373\371\202\377kfb/\362\303N"..., 16) = 16 
>>> clock_gettime(CLOCK_REALTIME, {1339187216, 902329405}) = 0 getuid32()
>>> = 0 geteuid32()                             = 0 
>>> rt_sigprocmask(SIG_SETMASK, ~[RTMIN RT_1], [], 8) = 0 mmap2(NULL,
>>> 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1,
>>> 0) = 0xb6e36000 mprotect(0xb6e36000, 4096, PROT_NONE)   = 0 
>>> clone(child_stack=0xb7634d64,
>>> flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED,
>>> parent_tidptr=0xb7635bd8, {entry_number:6, base_addr:0xb7635b70,
>>> limit:1048575, seg_32bit:1, contents:0, read_exec_only:0,
>>> limit_in_pages:1, seg_not_present:0, useable:1},
>>> child_tidptr=0xb7635bd8) = 3051 mmap2(NULL, 8388608,
>>> PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) =
>>> 0xb6636000 mprotect(0xb6636000, 4096, PROT_NONE)   = 0 
>>> clone(child_stack=0xb6e34d64,
>>> flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED,
>>> parent_tidptr=0xb6e35bd8, {entry_number:6, base_addr:0xb6e35b70,
>>> limit:1048575, seg_32bit:1, contents:0, read_exec_only:0,
>>> limit_in_pages:1, seg_not_present:0, useable:1},
>>> child_tidptr=0xb6e35bd8) = 3052 rt_sigprocmask(SIG_SETMASK, [], NULL,
>>> 8) = 0 gettimeofday({1339187216, 974869}, NULL) = 0 futex(0xb7836e30,
>>> FUTEX_WAIT_PRIVATE, 0, {2, 927460405}) = -1 ETIMEDOUT (Connection timed
>>> out) gettid()                                = 3050 write(2,
>>> "libust[3050/3050]: Error: Timed o"..., 107libust[3050/3050]: Error:
>>> Timed out waiting for ltt-sessiond (in lttng_ust_init() at
>>> lttng-ust-comm.c:912) ) = 107 futex(0xb7836e80, FUTEX_WAIT_PRIVATE, 2,
>>> NULL _______________________________________________ lttng-dev mailing
>>> list lttng-dev@lists.lttng.org 
>>> http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
>> 
>> 
>> -- Mathieu Desnoyers Operating System Efficiency R&D Consultant EfficiOS
>> Inc. http://www.efficios.com
> 
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.12 (GNU/Linux)

iQEcBAEBAgAGBQJP12V+AAoJEELoaioR9I02v3YH/A8x6NpChh1v3P0IxhHyMyQC
sJnfeejvCAxLWCSRm8hLgvaX65HfZA8nOcS4oPUEtonZU+QZS51no1W4biDUePKx
IPIzMy0+Fv5W/j6N0ocXgdP5lup2JiyPwqf13cj2ZC/n0BbGuuWrfpG4fYgwUaYG
TJSB5FrTF4KB5aQ6cYaMbsfv25mYGUWPGTvFjI3PFYPQ9MA6oimxBSYDMONM5IeI
5cZDEI7sd3BEO5Vf3YmymdxCWNwbvEOfyEJZo/TgsCV4ljyxthGWNvB1/meNdOqg
v8gaRyU29jXp/u58RP5fSuWrcpfQ4b4rRZ1dc1AXuUDD1vYBKrAY0t+3a1THJjE=
=ghI+
-----END PGP SIGNATURE-----

^ permalink raw reply	[flat|nested] 16+ messages in thread
* Re: UST Hanging at Tracepoint
       [not found]       ` <3F56B905CF2083408E8482044F20428AEF7A0AFE@ONWVEXCHMB01.ciena.com>
@ 2012-06-12 16:09         ` Burton, Michael
       [not found]         ` <3F56B905CF2083408E8482044F20428AEF7A0B4C@ONWVEXCHMB01.ciena.com>
  1 sibling, 0 replies; 16+ messages in thread
From: Burton, Michael @ 2012-06-12 16:09 UTC (permalink / raw)
  To: Burton, Michael, Mathieu Desnoyers, dgoulet; +Cc: lttng-dev

Also, I stepped through the code with gdb and things hang within the UST constructors and the main function of my application is never reached.

-----Original Message-----
From: Burton, Michael [mailto:mburton@ciena.com]
Sent: Tuesday, June 12, 2012 11:45 AM
To: Mathieu Desnoyers; dgoulet@efficios.com
Cc: lttng-dev@lists.lttng.org
Subject: Re: [lttng-dev] UST Hanging at Tracepoint

I put a fprintf at the beginning of main() and it never prints.

-----Original Message-----
From: Mathieu Desnoyers [mailto:mathieu.desnoyers@efficios.com]
Sent: Tuesday, June 12, 2012 10:14 AM
To: Burton, Michael; dgoulet@efficios.com
Cc: lttng-dev@lists.lttng.org
Subject: Re: [lttng-dev] UST Hanging at Tracepoint

* Mathieu Desnoyers (mathieu.desnoyers@efficios.com) wrote:
> Hi Michael,
>
> Please note that you should be able to run UST with either (or both) of:
>
> - root lttng-sessiond, for system-wide tracing
> - per-user lttng-sessiond, for per-user tracing.
>
> It is entirely normal that one of the two UST threads within the
> application can't find an active sessiond if, for instance, you have
> just a root (and no per-user) sessiond running.
>
> So although I'm glad that you got things working, I fail to understand
> your explanation: it should have worked fine with the root sessiond,
> even if no ~/.lttng exists.
>
> David, can you look into this ?

hrm, further insight: is your program _really_ hang ? Try to make sure
you put a fprintf(stderr, ".....") at the beginning of your main().
Looking once more at your detailed UST output indicates that the
per-user UST thread tells you that it cannot see any running sessiond.
So yes, this thread blocks, but the thread talking to the root sessiond
can very well be working fine, and your application might also be
running fine. This should be double-checked.

Thanks,

Mathieu

>
> Thanks,
>
> Mathieu
>
> * Burton, Michael (mburton@ciena.com) wrote:
> > Mathieu,
> >
> > I figured out the problem.  When we start sessiond on our system as root, the /.lttng/ folder is not created, along with its contents.  UST timed out since there was no sock_path.  I was able to get UST working once I got sessiond running.
> >
> > Thanks.
> > Michael
> >
> > ________________________________________
> > From: Mathieu Desnoyers [mathieu.desnoyers@efficios.com]
> > Sent: 08 June 2012 21:38
> > To: Burton, Michael
> > Cc: lttng-dev@lists.lttng.org
> > Subject: Re: [lttng-dev] UST Hanging at Tracepoint
> >
> > * Burton, Michael (mburton@ciena.com) wrote:
> > > For further insight, I've attached the strace from the hanging
> > > program.  I also upgraded LTTng-UST to 2.0.3 and userspace-rcu to
> > > 0.7.3.
> >
> > Haven't had time to look at it yet, but could you also provide the
> > output of:
> >
> > LTTNG_UST_DEBUG=1 youprogname
> >
> > (starting your UST instrumented program with LTTNG_UST_DEBUG=1 env. var.
> > set)
> >
> > Thanks!
> >
> > Mathieu
> >
> > >
> > > Michael
> > >
> > > From: Burton, Michael [mailto:mburton@ciena.com]
> > > Sent: Thursday, June 07, 2012 2:16 PM
> > > To: lttng-dev@lists.lttng.org
> > > Subject: [lttng-dev] UST Hanging at Tracepoint
> > >
> > > I am working on getting LTTng 2.0 working in our 2.6.35 kernel.  I have kernel tracing working but I'm having problems with UST.
> > >
> > > I have a program with a dynamic UST tracepoint.  When I run the program with all UST events enabled (lttng enable-event -a -u) the program hangs on the tracepoint.  The last line from the UST debug is:
> > >
> > > libust[6184/6185]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)
> > >
> > > Any insight into why this is happening?  I've attached the UST and lttng-sessiond debug generated by the program.
> > >
> > > I am running the following:
> > > lttng-modules-2.6.32 (found through this mailing list)
> > > lttng-tools-2.0.1
> > > lttng-ust-2.0.2
> > > userspace-rcu-0.7.0
> > >
> > > Thanks,
> > > Michael
> >
> > Content-Description: strace_debug.txt
> > > execve("/ciena/bin/idp", ["idp", "help"], [/* 11 vars */]) = 0
> > > brk(0)                                  = 0x804e000
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb783c000
> > > access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
> > > open("/etc/ld.so.cache", O_RDONLY)      = 3
> > > fstat64(3, {st_mode=S_IFREG|0644, st_size=17420, ...}) = 0
> > > mmap2(NULL, 17420, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7837000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220A\0\0004\0\0\0$"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=209876, ...}) = 0
> > > mmap2(NULL, 212940, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7803000
> > > mmap2(0xb7832000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2e) = 0xb7832000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust-ctl.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200-\0\0004\0\0\0\274"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=140020, ...}) = 0
> > > mmap2(NULL, 142828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77e0000
> > > mmap2(0xb7802000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21) = 0xb7802000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust-fork.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\5\0\0004\0\0\0X"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=3864, ...}) = 0
> > > mmap2(NULL, 6820, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77de000
> > > mmap2(0xb77df000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77df000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust-tracepoint.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220\v\0\0004\0\0\0008"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=16632, ...}) = 0
> > > mmap2(NULL, 19944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d9000
> > > mmap2(0xb77dd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xb77dd000
> > > close(3)                                = 0
> > > open("/usr/lib/liblttng-ust-libc-wrapper.so.0", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\t\0\0004\0\0\0\364"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=9300, ...}) = 0
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77d8000
> > > mmap2(NULL, 12104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d5000
> > > mmap2(0xb77d7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77d7000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d0000
> > > mmap2(0xb77d4000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77d4000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-common.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\5\0\0004\0\0\0\34"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=5084, ...}) = 0
> > > mmap2(NULL, 8032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ce000
> > > mmap2(0xb77cf000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77cf000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-qsbr.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\22\0\0004\0\0\0\374"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c9000
> > > mmap2(0xb77cd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77cd000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-mb.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c4000
> > > mmap2(0xb77c8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c8000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-signal.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\23\0\0004\0\0\0\10"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18712, ...}) = 0
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77c3000
> > > mmap2(NULL, 21704, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77bd000
> > > mmap2(0xb77c2000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c2000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-cds.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\f\0\0004\0\0\0t"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=26204, ...}) = 0
> > > mmap2(NULL, 25004, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b6000
> > > mmap2(0xb77bc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb77bc000
> > > close(3)                                = 0
> > > open("/usr/lib/liburcu-bp.so.1", O_RDONLY) = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\24\0\0004\0\0\0\360"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=19968, ...}) = 0
> > > mmap2(NULL, 23128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b0000
> > > mmap2(0xb77b5000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77b5000
> > > close(3)                                = 0
> > > open("/lib/libdl.so.2", O_RDONLY)       = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \n\0\0004\0\0\0<"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=9668, ...}) = 0
> > > mmap2(NULL, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ac000
> > > mmap2(0xb77ae000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77ae000
> > > close(3)                                = 0
> > > open("/lib/libuuid.so.1", O_RDONLY)     = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@\r\0\0004\0\0\0\350"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=12872, ...}) = 0
> > > mmap2(NULL, 15632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77a8000
> > > mmap2(0xb77ab000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2) = 0xb77ab000
> > > close(3)                                = 0
> > > open("/lib/libc.so.6", O_RDONLY)        = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220n\1\0004\0\0\0L"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=1347780, ...}) = 0
> > > mmap2(NULL, 1358120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb765c000
> > > mprotect(0xb77a1000, 4096, PROT_NONE)   = 0
> > > mmap2(0xb77a2000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x145) = 0xb77a2000
> > > mmap2(0xb77a5000, 10536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb77a5000
> > > close(3)                                = 0
> > > open("/lib/librt.so.1", O_RDONLY)       = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\30\0\0004\0\0\0\230"..., 512) = 512
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb765b000
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=30616, ...}) = 0
> > > mmap2(NULL, 33364, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7652000
> > > mmap2(0xb7659000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7659000
> > > close(3)                                = 0
> > > open("/lib/libpthread.so.0", O_RDONLY)  = 3
> > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360J\0\0004\0\0\0\354"..., 512) = 512
> > > fstat64(3, {st_mode=S_IFREG|0755, st_size=88420, ...}) = 0
> > > mmap2(NULL, 98828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7639000
> > > mmap2(0xb764e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0xb764e000
> > > mmap2(0xb7650000, 4620, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7650000
> > > close(3)                                = 0
> > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7638000
> > > mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7636000
> > > set_thread_area({entry_number:-1 -> 6, base_addr:0xb7636d80, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
> > > mprotect(0xb764e000, 4096, PROT_READ)   = 0
> > > mprotect(0xb7659000, 4096, PROT_READ)   = 0
> > > mprotect(0xb77a2000, 8192, PROT_READ)   = 0
> > > mprotect(0xb77ae000, 4096, PROT_READ)   = 0
> > > mprotect(0xb785a000, 4096, PROT_READ)   = 0
> > > munmap(0xb7837000, 17420)               = 0
> > > set_tid_address(0xb7636de8)             = 3050
> > > set_robust_list(0xb7636df0, 0xc)        = 0
> > > futex(0xbfaaf560, FUTEX_WAKE_PRIVATE, 1) = 0
> > > futex(0xbfaaf560, 0x189 /* FUTEX_??? */, 1, NULL, bfaaf570) = -1 EAGAIN (Resource temporarily unavailable)
> > > rt_sigaction(SIGRTMIN, {0xb763d4f0, [], SA_SIGINFO}, NULL, 8) = 0
> > > rt_sigaction(SIGRT_1, {0xb763d9c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > > rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> > > getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
> > > uname({sys="Linux", node="5410", ...})  = 0
> > > rt_sigaction(SIGUSR1, {0xb77bec0a, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > > futex(0xb77af06c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
> > > brk(0)                                  = 0x804e000
> > > brk(0x806f000)                          = 0x806f000
> > > gettimeofday({1339187216, 880428}, NULL) = 0
> > > open("/dev/urandom", O_RDONLY|O_LARGEFILE) = 3
> > > fcntl64(3, F_GETFD)                     = 0
> > > fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
> > > getuid32()                              = 0
> > > gettimeofday({1339187216, 889603}, NULL) = 0
> > > gettimeofday({1339187216, 894471}, NULL) = 0
> > > read(3, "\275\0051F\215\373\371\202\377kfb/\362\303N"..., 16) = 16
> > > clock_gettime(CLOCK_REALTIME, {1339187216, 902329405}) = 0
> > > getuid32()                              = 0
> > > geteuid32()                             = 0
> > > rt_sigprocmask(SIG_SETMASK, ~[RTMIN RT_1], [], 8) = 0
> > > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6e36000
> > > mprotect(0xb6e36000, 4096, PROT_NONE)   = 0
> > > clone(child_stack=0xb7634d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb7635bd8, {entry_number:6, base_addr:0xb7635b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb7635bd8) = 3051
> > > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6636000
> > > mprotect(0xb6636000, 4096, PROT_NONE)   = 0
> > > clone(child_stack=0xb6e34d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb6e35bd8, {entry_number:6, base_addr:0xb6e35b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb6e35bd8) = 3052
> > > rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
> > > gettimeofday({1339187216, 974869}, NULL) = 0
> > > futex(0xb7836e30, FUTEX_WAIT_PRIVATE, 0, {2, 927460405}) = -1 ETIMEDOUT (Connection timed out)
> > > gettid()                                = 3050
> > > write(2, "libust[3050/3050]: Error: Timed o"..., 107libust[3050/3050]: Error: Timed out waiting for ltt-sessiond (in lttng_ust_init() at lttng-ust-comm.c:912)
> > > ) = 107
> > > futex(0xb7836e80, FUTEX_WAIT_PRIVATE, 2, NULL
> > > _______________________________________________
> > > lttng-dev mailing list
> > > lttng-dev@lists.lttng.org
> > > http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
> >
> >
> > --
> > Mathieu Desnoyers
> > Operating System Efficiency R&D Consultant
> > EfficiOS Inc.
> > http://www.efficios.com
>
> --
> Mathieu Desnoyers
> Operating System Efficiency R&D Consultant
> EfficiOS Inc.
> http://www.efficios.com
>
> _______________________________________________
> lttng-dev mailing list
> lttng-dev@lists.lttng.org
> http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

_______________________________________________
lttng-dev mailing list
lttng-dev@lists.lttng.org
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

^ permalink raw reply	[flat|nested] 16+ messages in thread
* Re: UST Hanging at Tracepoint
       [not found]         ` <3F56B905CF2083408E8482044F20428AEF7A0B4C@ONWVEXCHMB01.ciena.com>
@ 2012-06-12 16:25           ` Mathieu Desnoyers
       [not found]           ` <20120612162526.GB16098@Krystal>
  1 sibling, 0 replies; 16+ messages in thread
From: Mathieu Desnoyers @ 2012-06-12 16:25 UTC (permalink / raw)
  To: Burton, Michael; +Cc: lttng-dev, dgoulet

Please build you app and lttng-ust with:

make CFLAGS=-g   (so with -O0 optimisation, and debug info)

then, under gdb, please provide the full backtrace of each thread when
stucked with:

thread apply all bt full

Thanks,

Mathieu


* Burton, Michael (mburton@ciena.com) wrote:
> Also, I stepped through the code with gdb and things hang within the UST constructors and the main function of my application is never reached.
> 
> -----Original Message-----
> From: Burton, Michael [mailto:mburton@ciena.com]
> Sent: Tuesday, June 12, 2012 11:45 AM
> To: Mathieu Desnoyers; dgoulet@efficios.com
> Cc: lttng-dev@lists.lttng.org
> Subject: Re: [lttng-dev] UST Hanging at Tracepoint
> 
> I put a fprintf at the beginning of main() and it never prints.
> 
> -----Original Message-----
> From: Mathieu Desnoyers [mailto:mathieu.desnoyers@efficios.com]
> Sent: Tuesday, June 12, 2012 10:14 AM
> To: Burton, Michael; dgoulet@efficios.com
> Cc: lttng-dev@lists.lttng.org
> Subject: Re: [lttng-dev] UST Hanging at Tracepoint
> 
> * Mathieu Desnoyers (mathieu.desnoyers@efficios.com) wrote:
> > Hi Michael,
> >
> > Please note that you should be able to run UST with either (or both) of:
> >
> > - root lttng-sessiond, for system-wide tracing
> > - per-user lttng-sessiond, for per-user tracing.
> >
> > It is entirely normal that one of the two UST threads within the
> > application can't find an active sessiond if, for instance, you have
> > just a root (and no per-user) sessiond running.
> >
> > So although I'm glad that you got things working, I fail to understand
> > your explanation: it should have worked fine with the root sessiond,
> > even if no ~/.lttng exists.
> >
> > David, can you look into this ?
> 
> hrm, further insight: is your program _really_ hang ? Try to make sure
> you put a fprintf(stderr, ".....") at the beginning of your main().
> Looking once more at your detailed UST output indicates that the
> per-user UST thread tells you that it cannot see any running sessiond.
> So yes, this thread blocks, but the thread talking to the root sessiond
> can very well be working fine, and your application might also be
> running fine. This should be double-checked.
> 
> Thanks,
> 
> Mathieu
> 
> >
> > Thanks,
> >
> > Mathieu
> >
> > * Burton, Michael (mburton@ciena.com) wrote:
> > > Mathieu,
> > >
> > > I figured out the problem.  When we start sessiond on our system as root, the /.lttng/ folder is not created, along with its contents.  UST timed out since there was no sock_path.  I was able to get UST working once I got sessiond running.
> > >
> > > Thanks.
> > > Michael
> > >
> > > ________________________________________
> > > From: Mathieu Desnoyers [mathieu.desnoyers@efficios.com]
> > > Sent: 08 June 2012 21:38
> > > To: Burton, Michael
> > > Cc: lttng-dev@lists.lttng.org
> > > Subject: Re: [lttng-dev] UST Hanging at Tracepoint
> > >
> > > * Burton, Michael (mburton@ciena.com) wrote:
> > > > For further insight, I've attached the strace from the hanging
> > > > program.  I also upgraded LTTng-UST to 2.0.3 and userspace-rcu to
> > > > 0.7.3.
> > >
> > > Haven't had time to look at it yet, but could you also provide the
> > > output of:
> > >
> > > LTTNG_UST_DEBUG=1 youprogname
> > >
> > > (starting your UST instrumented program with LTTNG_UST_DEBUG=1 env. var.
> > > set)
> > >
> > > Thanks!
> > >
> > > Mathieu
> > >
> > > >
> > > > Michael
> > > >
> > > > From: Burton, Michael [mailto:mburton@ciena.com]
> > > > Sent: Thursday, June 07, 2012 2:16 PM
> > > > To: lttng-dev@lists.lttng.org
> > > > Subject: [lttng-dev] UST Hanging at Tracepoint
> > > >
> > > > I am working on getting LTTng 2.0 working in our 2.6.35 kernel.  I have kernel tracing working but I'm having problems with UST.
> > > >
> > > > I have a program with a dynamic UST tracepoint.  When I run the program with all UST events enabled (lttng enable-event -a -u) the program hangs on the tracepoint.  The last line from the UST debug is:
> > > >
> > > > libust[6184/6185]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)
> > > >
> > > > Any insight into why this is happening?  I've attached the UST and lttng-sessiond debug generated by the program.
> > > >
> > > > I am running the following:
> > > > lttng-modules-2.6.32 (found through this mailing list)
> > > > lttng-tools-2.0.1
> > > > lttng-ust-2.0.2
> > > > userspace-rcu-0.7.0
> > > >
> > > > Thanks,
> > > > Michael
> > >
> > > Content-Description: strace_debug.txt
> > > > execve("/ciena/bin/idp", ["idp", "help"], [/* 11 vars */]) = 0
> > > > brk(0)                                  = 0x804e000
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb783c000
> > > > access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
> > > > open("/etc/ld.so.cache", O_RDONLY)      = 3
> > > > fstat64(3, {st_mode=S_IFREG|0644, st_size=17420, ...}) = 0
> > > > mmap2(NULL, 17420, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7837000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220A\0\0004\0\0\0$"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=209876, ...}) = 0
> > > > mmap2(NULL, 212940, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7803000
> > > > mmap2(0xb7832000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2e) = 0xb7832000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust-ctl.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200-\0\0004\0\0\0\274"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=140020, ...}) = 0
> > > > mmap2(NULL, 142828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77e0000
> > > > mmap2(0xb7802000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21) = 0xb7802000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust-fork.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\5\0\0004\0\0\0X"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=3864, ...}) = 0
> > > > mmap2(NULL, 6820, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77de000
> > > > mmap2(0xb77df000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77df000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust-tracepoint.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220\v\0\0004\0\0\0008"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=16632, ...}) = 0
> > > > mmap2(NULL, 19944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d9000
> > > > mmap2(0xb77dd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xb77dd000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust-libc-wrapper.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\t\0\0004\0\0\0\364"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=9300, ...}) = 0
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77d8000
> > > > mmap2(NULL, 12104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d5000
> > > > mmap2(0xb77d7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77d7000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d0000
> > > > mmap2(0xb77d4000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77d4000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-common.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\5\0\0004\0\0\0\34"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=5084, ...}) = 0
> > > > mmap2(NULL, 8032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ce000
> > > > mmap2(0xb77cf000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77cf000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-qsbr.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\22\0\0004\0\0\0\374"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c9000
> > > > mmap2(0xb77cd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77cd000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-mb.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c4000
> > > > mmap2(0xb77c8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c8000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-signal.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\23\0\0004\0\0\0\10"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18712, ...}) = 0
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77c3000
> > > > mmap2(NULL, 21704, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77bd000
> > > > mmap2(0xb77c2000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c2000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-cds.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\f\0\0004\0\0\0t"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=26204, ...}) = 0
> > > > mmap2(NULL, 25004, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b6000
> > > > mmap2(0xb77bc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb77bc000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-bp.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\24\0\0004\0\0\0\360"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=19968, ...}) = 0
> > > > mmap2(NULL, 23128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b0000
> > > > mmap2(0xb77b5000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77b5000
> > > > close(3)                                = 0
> > > > open("/lib/libdl.so.2", O_RDONLY)       = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \n\0\0004\0\0\0<"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=9668, ...}) = 0
> > > > mmap2(NULL, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ac000
> > > > mmap2(0xb77ae000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77ae000
> > > > close(3)                                = 0
> > > > open("/lib/libuuid.so.1", O_RDONLY)     = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@\r\0\0004\0\0\0\350"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=12872, ...}) = 0
> > > > mmap2(NULL, 15632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77a8000
> > > > mmap2(0xb77ab000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2) = 0xb77ab000
> > > > close(3)                                = 0
> > > > open("/lib/libc.so.6", O_RDONLY)        = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220n\1\0004\0\0\0L"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=1347780, ...}) = 0
> > > > mmap2(NULL, 1358120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb765c000
> > > > mprotect(0xb77a1000, 4096, PROT_NONE)   = 0
> > > > mmap2(0xb77a2000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x145) = 0xb77a2000
> > > > mmap2(0xb77a5000, 10536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb77a5000
> > > > close(3)                                = 0
> > > > open("/lib/librt.so.1", O_RDONLY)       = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\30\0\0004\0\0\0\230"..., 512) = 512
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb765b000
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=30616, ...}) = 0
> > > > mmap2(NULL, 33364, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7652000
> > > > mmap2(0xb7659000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7659000
> > > > close(3)                                = 0
> > > > open("/lib/libpthread.so.0", O_RDONLY)  = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360J\0\0004\0\0\0\354"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=88420, ...}) = 0
> > > > mmap2(NULL, 98828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7639000
> > > > mmap2(0xb764e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0xb764e000
> > > > mmap2(0xb7650000, 4620, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7650000
> > > > close(3)                                = 0
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7638000
> > > > mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7636000
> > > > set_thread_area({entry_number:-1 -> 6, base_addr:0xb7636d80, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
> > > > mprotect(0xb764e000, 4096, PROT_READ)   = 0
> > > > mprotect(0xb7659000, 4096, PROT_READ)   = 0
> > > > mprotect(0xb77a2000, 8192, PROT_READ)   = 0
> > > > mprotect(0xb77ae000, 4096, PROT_READ)   = 0
> > > > mprotect(0xb785a000, 4096, PROT_READ)   = 0
> > > > munmap(0xb7837000, 17420)               = 0
> > > > set_tid_address(0xb7636de8)             = 3050
> > > > set_robust_list(0xb7636df0, 0xc)        = 0
> > > > futex(0xbfaaf560, FUTEX_WAKE_PRIVATE, 1) = 0
> > > > futex(0xbfaaf560, 0x189 /* FUTEX_??? */, 1, NULL, bfaaf570) = -1 EAGAIN (Resource temporarily unavailable)
> > > > rt_sigaction(SIGRTMIN, {0xb763d4f0, [], SA_SIGINFO}, NULL, 8) = 0
> > > > rt_sigaction(SIGRT_1, {0xb763d9c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > > > rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> > > > getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
> > > > uname({sys="Linux", node="5410", ...})  = 0
> > > > rt_sigaction(SIGUSR1, {0xb77bec0a, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > > > futex(0xb77af06c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
> > > > brk(0)                                  = 0x804e000
> > > > brk(0x806f000)                          = 0x806f000
> > > > gettimeofday({1339187216, 880428}, NULL) = 0
> > > > open("/dev/urandom", O_RDONLY|O_LARGEFILE) = 3
> > > > fcntl64(3, F_GETFD)                     = 0
> > > > fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
> > > > getuid32()                              = 0
> > > > gettimeofday({1339187216, 889603}, NULL) = 0
> > > > gettimeofday({1339187216, 894471}, NULL) = 0
> > > > read(3, "\275\0051F\215\373\371\202\377kfb/\362\303N"..., 16) = 16
> > > > clock_gettime(CLOCK_REALTIME, {1339187216, 902329405}) = 0
> > > > getuid32()                              = 0
> > > > geteuid32()                             = 0
> > > > rt_sigprocmask(SIG_SETMASK, ~[RTMIN RT_1], [], 8) = 0
> > > > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6e36000
> > > > mprotect(0xb6e36000, 4096, PROT_NONE)   = 0
> > > > clone(child_stack=0xb7634d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb7635bd8, {entry_number:6, base_addr:0xb7635b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb7635bd8) = 3051
> > > > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6636000
> > > > mprotect(0xb6636000, 4096, PROT_NONE)   = 0
> > > > clone(child_stack=0xb6e34d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb6e35bd8, {entry_number:6, base_addr:0xb6e35b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb6e35bd8) = 3052
> > > > rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
> > > > gettimeofday({1339187216, 974869}, NULL) = 0
> > > > futex(0xb7836e30, FUTEX_WAIT_PRIVATE, 0, {2, 927460405}) = -1 ETIMEDOUT (Connection timed out)
> > > > gettid()                                = 3050
> > > > write(2, "libust[3050/3050]: Error: Timed o"..., 107libust[3050/3050]: Error: Timed out waiting for ltt-sessiond (in lttng_ust_init() at lttng-ust-comm.c:912)
> > > > ) = 107
> > > > futex(0xb7836e80, FUTEX_WAIT_PRIVATE, 2, NULL
> > > > _______________________________________________
> > > > lttng-dev mailing list
> > > > lttng-dev@lists.lttng.org
> > > > http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
> > >
> > >
> > > --
> > > Mathieu Desnoyers
> > > Operating System Efficiency R&D Consultant
> > > EfficiOS Inc.
> > > http://www.efficios.com
> >
> > --
> > Mathieu Desnoyers
> > Operating System Efficiency R&D Consultant
> > EfficiOS Inc.
> > http://www.efficios.com
> >
> > _______________________________________________
> > lttng-dev mailing list
> > lttng-dev@lists.lttng.org
> > http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
> 
> --
> Mathieu Desnoyers
> Operating System Efficiency R&D Consultant
> EfficiOS Inc.
> http://www.efficios.com
> 
> _______________________________________________
> lttng-dev mailing list
> lttng-dev@lists.lttng.org
> http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

-- 
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

^ permalink raw reply	[flat|nested] 16+ messages in thread
* Re: UST Hanging at Tracepoint
       [not found]           ` <20120612162526.GB16098@Krystal>
@ 2012-06-12 18:24             ` Burton, Michael
  2012-06-12 22:25             ` Burton, Michael
       [not found]             ` <3F56B905CF2083408E8482044F20428AEF7A0ED6@ONWVEXCHMB01.ciena.com>
  2 siblings, 0 replies; 16+ messages in thread
From: Burton, Michael @ 2012-06-12 18:24 UTC (permalink / raw)
  To: Mathieu Desnoyers; +Cc: lttng-dev, dgoulet

[-- Attachment #1: Type: text/plain, Size: 19342 bytes --]

I've attached the full bt.

Michael

-----Original Message-----
From: Mathieu Desnoyers [mailto:mathieu.desnoyers@efficios.com]
Sent: Tuesday, June 12, 2012 12:25 PM
To: Burton, Michael
Cc: dgoulet@efficios.com; lttng-dev@lists.lttng.org
Subject: Re: [lttng-dev] UST Hanging at Tracepoint

Please build you app and lttng-ust with:

make CFLAGS=-g   (so with -O0 optimisation, and debug info)

then, under gdb, please provide the full backtrace of each thread when
stucked with:

thread apply all bt full

Thanks,

Mathieu


* Burton, Michael (mburton@ciena.com) wrote:
> Also, I stepped through the code with gdb and things hang within the UST constructors and the main function of my application is never reached.
>
> -----Original Message-----
> From: Burton, Michael [mailto:mburton@ciena.com]
> Sent: Tuesday, June 12, 2012 11:45 AM
> To: Mathieu Desnoyers; dgoulet@efficios.com
> Cc: lttng-dev@lists.lttng.org
> Subject: Re: [lttng-dev] UST Hanging at Tracepoint
>
> I put a fprintf at the beginning of main() and it never prints.
>
> -----Original Message-----
> From: Mathieu Desnoyers [mailto:mathieu.desnoyers@efficios.com]
> Sent: Tuesday, June 12, 2012 10:14 AM
> To: Burton, Michael; dgoulet@efficios.com
> Cc: lttng-dev@lists.lttng.org
> Subject: Re: [lttng-dev] UST Hanging at Tracepoint
>
> * Mathieu Desnoyers (mathieu.desnoyers@efficios.com) wrote:
> > Hi Michael,
> >
> > Please note that you should be able to run UST with either (or both) of:
> >
> > - root lttng-sessiond, for system-wide tracing
> > - per-user lttng-sessiond, for per-user tracing.
> >
> > It is entirely normal that one of the two UST threads within the
> > application can't find an active sessiond if, for instance, you have
> > just a root (and no per-user) sessiond running.
> >
> > So although I'm glad that you got things working, I fail to understand
> > your explanation: it should have worked fine with the root sessiond,
> > even if no ~/.lttng exists.
> >
> > David, can you look into this ?
>
> hrm, further insight: is your program _really_ hang ? Try to make sure
> you put a fprintf(stderr, ".....") at the beginning of your main().
> Looking once more at your detailed UST output indicates that the
> per-user UST thread tells you that it cannot see any running sessiond.
> So yes, this thread blocks, but the thread talking to the root sessiond
> can very well be working fine, and your application might also be
> running fine. This should be double-checked.
>
> Thanks,
>
> Mathieu
>
> >
> > Thanks,
> >
> > Mathieu
> >
> > * Burton, Michael (mburton@ciena.com) wrote:
> > > Mathieu,
> > >
> > > I figured out the problem.  When we start sessiond on our system as root, the /.lttng/ folder is not created, along with its contents.  UST timed out since there was no sock_path.  I was able to get UST working once I got sessiond running.
> > >
> > > Thanks.
> > > Michael
> > >
> > > ________________________________________
> > > From: Mathieu Desnoyers [mathieu.desnoyers@efficios.com]
> > > Sent: 08 June 2012 21:38
> > > To: Burton, Michael
> > > Cc: lttng-dev@lists.lttng.org
> > > Subject: Re: [lttng-dev] UST Hanging at Tracepoint
> > >
> > > * Burton, Michael (mburton@ciena.com) wrote:
> > > > For further insight, I've attached the strace from the hanging
> > > > program.  I also upgraded LTTng-UST to 2.0.3 and userspace-rcu to
> > > > 0.7.3.
> > >
> > > Haven't had time to look at it yet, but could you also provide the
> > > output of:
> > >
> > > LTTNG_UST_DEBUG=1 youprogname
> > >
> > > (starting your UST instrumented program with LTTNG_UST_DEBUG=1 env. var.
> > > set)
> > >
> > > Thanks!
> > >
> > > Mathieu
> > >
> > > >
> > > > Michael
> > > >
> > > > From: Burton, Michael [mailto:mburton@ciena.com]
> > > > Sent: Thursday, June 07, 2012 2:16 PM
> > > > To: lttng-dev@lists.lttng.org
> > > > Subject: [lttng-dev] UST Hanging at Tracepoint
> > > >
> > > > I am working on getting LTTng 2.0 working in our 2.6.35 kernel.  I have kernel tracing working but I'm having problems with UST.
> > > >
> > > > I have a program with a dynamic UST tracepoint.  When I run the program with all UST events enabled (lttng enable-event -a -u) the program hangs on the tracepoint.  The last line from the UST debug is:
> > > >
> > > > libust[6184/6185]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)
> > > >
> > > > Any insight into why this is happening?  I've attached the UST and lttng-sessiond debug generated by the program.
> > > >
> > > > I am running the following:
> > > > lttng-modules-2.6.32 (found through this mailing list)
> > > > lttng-tools-2.0.1
> > > > lttng-ust-2.0.2
> > > > userspace-rcu-0.7.0
> > > >
> > > > Thanks,
> > > > Michael
> > >
> > > Content-Description: strace_debug.txt
> > > > execve("/ciena/bin/idp", ["idp", "help"], [/* 11 vars */]) = 0
> > > > brk(0)                                  = 0x804e000
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb783c000
> > > > access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
> > > > open("/etc/ld.so.cache", O_RDONLY)      = 3
> > > > fstat64(3, {st_mode=S_IFREG|0644, st_size=17420, ...}) = 0
> > > > mmap2(NULL, 17420, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7837000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220A\0\0004\0\0\0$"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=209876, ...}) = 0
> > > > mmap2(NULL, 212940, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7803000
> > > > mmap2(0xb7832000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2e) = 0xb7832000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust-ctl.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200-\0\0004\0\0\0\274"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=140020, ...}) = 0
> > > > mmap2(NULL, 142828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77e0000
> > > > mmap2(0xb7802000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21) = 0xb7802000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust-fork.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\5\0\0004\0\0\0X"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=3864, ...}) = 0
> > > > mmap2(NULL, 6820, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77de000
> > > > mmap2(0xb77df000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77df000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust-tracepoint.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220\v\0\0004\0\0\0008"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=16632, ...}) = 0
> > > > mmap2(NULL, 19944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d9000
> > > > mmap2(0xb77dd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xb77dd000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust-libc-wrapper.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\t\0\0004\0\0\0\364"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=9300, ...}) = 0
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77d8000
> > > > mmap2(NULL, 12104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d5000
> > > > mmap2(0xb77d7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77d7000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d0000
> > > > mmap2(0xb77d4000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77d4000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-common.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\5\0\0004\0\0\0\34"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=5084, ...}) = 0
> > > > mmap2(NULL, 8032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ce000
> > > > mmap2(0xb77cf000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77cf000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-qsbr.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\22\0\0004\0\0\0\374"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c9000
> > > > mmap2(0xb77cd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77cd000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-mb.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c4000
> > > > mmap2(0xb77c8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c8000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-signal.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\23\0\0004\0\0\0\10"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18712, ...}) = 0
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77c3000
> > > > mmap2(NULL, 21704, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77bd000
> > > > mmap2(0xb77c2000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c2000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-cds.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\f\0\0004\0\0\0t"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=26204, ...}) = 0
> > > > mmap2(NULL, 25004, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b6000
> > > > mmap2(0xb77bc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb77bc000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-bp.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\24\0\0004\0\0\0\360"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=19968, ...}) = 0
> > > > mmap2(NULL, 23128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b0000
> > > > mmap2(0xb77b5000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77b5000
> > > > close(3)                                = 0
> > > > open("/lib/libdl.so.2", O_RDONLY)       = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \n\0\0004\0\0\0<"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=9668, ...}) = 0
> > > > mmap2(NULL, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ac000
> > > > mmap2(0xb77ae000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77ae000
> > > > close(3)                                = 0
> > > > open("/lib/libuuid.so.1", O_RDONLY)     = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@\r\0\0004\0\0\0\350"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=12872, ...}) = 0
> > > > mmap2(NULL, 15632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77a8000
> > > > mmap2(0xb77ab000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2) = 0xb77ab000
> > > > close(3)                                = 0
> > > > open("/lib/libc.so.6", O_RDONLY)        = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220n\1\0004\0\0\0L"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=1347780, ...}) = 0
> > > > mmap2(NULL, 1358120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb765c000
> > > > mprotect(0xb77a1000, 4096, PROT_NONE)   = 0
> > > > mmap2(0xb77a2000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x145) = 0xb77a2000
> > > > mmap2(0xb77a5000, 10536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb77a5000
> > > > close(3)                                = 0
> > > > open("/lib/librt.so.1", O_RDONLY)       = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\30\0\0004\0\0\0\230"..., 512) = 512
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb765b000
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=30616, ...}) = 0
> > > > mmap2(NULL, 33364, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7652000
> > > > mmap2(0xb7659000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7659000
> > > > close(3)                                = 0
> > > > open("/lib/libpthread.so.0", O_RDONLY)  = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360J\0\0004\0\0\0\354"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=88420, ...}) = 0
> > > > mmap2(NULL, 98828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7639000
> > > > mmap2(0xb764e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0xb764e000
> > > > mmap2(0xb7650000, 4620, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7650000
> > > > close(3)                                = 0
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7638000
> > > > mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7636000
> > > > set_thread_area({entry_number:-1 -> 6, base_addr:0xb7636d80, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
> > > > mprotect(0xb764e000, 4096, PROT_READ)   = 0
> > > > mprotect(0xb7659000, 4096, PROT_READ)   = 0
> > > > mprotect(0xb77a2000, 8192, PROT_READ)   = 0
> > > > mprotect(0xb77ae000, 4096, PROT_READ)   = 0
> > > > mprotect(0xb785a000, 4096, PROT_READ)   = 0
> > > > munmap(0xb7837000, 17420)               = 0
> > > > set_tid_address(0xb7636de8)             = 3050
> > > > set_robust_list(0xb7636df0, 0xc)        = 0
> > > > futex(0xbfaaf560, FUTEX_WAKE_PRIVATE, 1) = 0
> > > > futex(0xbfaaf560, 0x189 /* FUTEX_??? */, 1, NULL, bfaaf570) = -1 EAGAIN (Resource temporarily unavailable)
> > > > rt_sigaction(SIGRTMIN, {0xb763d4f0, [], SA_SIGINFO}, NULL, 8) = 0
> > > > rt_sigaction(SIGRT_1, {0xb763d9c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > > > rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> > > > getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
> > > > uname({sys="Linux", node="5410", ...})  = 0
> > > > rt_sigaction(SIGUSR1, {0xb77bec0a, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > > > futex(0xb77af06c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
> > > > brk(0)                                  = 0x804e000
> > > > brk(0x806f000)                          = 0x806f000
> > > > gettimeofday({1339187216, 880428}, NULL) = 0
> > > > open("/dev/urandom", O_RDONLY|O_LARGEFILE) = 3
> > > > fcntl64(3, F_GETFD)                     = 0
> > > > fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
> > > > getuid32()                              = 0
> > > > gettimeofday({1339187216, 889603}, NULL) = 0
> > > > gettimeofday({1339187216, 894471}, NULL) = 0
> > > > read(3, "\275\0051F\215\373\371\202\377kfb/\362\303N"..., 16) = 16
> > > > clock_gettime(CLOCK_REALTIME, {1339187216, 902329405}) = 0
> > > > getuid32()                              = 0
> > > > geteuid32()                             = 0
> > > > rt_sigprocmask(SIG_SETMASK, ~[RTMIN RT_1], [], 8) = 0
> > > > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6e36000
> > > > mprotect(0xb6e36000, 4096, PROT_NONE)   = 0
> > > > clone(child_stack=0xb7634d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb7635bd8, {entry_number:6, base_addr:0xb7635b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb7635bd8) = 3051
> > > > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6636000
> > > > mprotect(0xb6636000, 4096, PROT_NONE)   = 0
> > > > clone(child_stack=0xb6e34d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb6e35bd8, {entry_number:6, base_addr:0xb6e35b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb6e35bd8) = 3052
> > > > rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
> > > > gettimeofday({1339187216, 974869}, NULL) = 0
> > > > futex(0xb7836e30, FUTEX_WAIT_PRIVATE, 0, {2, 927460405}) = -1 ETIMEDOUT (Connection timed out)
> > > > gettid()                                = 3050
> > > > write(2, "libust[3050/3050]: Error: Timed o"..., 107libust[3050/3050]: Error: Timed out waiting for ltt-sessiond (in lttng_ust_init() at lttng-ust-comm.c:912)
> > > > ) = 107
> > > > futex(0xb7836e80, FUTEX_WAIT_PRIVATE, 2, NULL
> > > > _______________________________________________
> > > > lttng-dev mailing list
> > > > lttng-dev@lists.lttng.org
> > > > http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
> > >
> > >
> > > --
> > > Mathieu Desnoyers
> > > Operating System Efficiency R&D Consultant
> > > EfficiOS Inc.
> > > http://www.efficios.com
> >
> > --
> > Mathieu Desnoyers
> > Operating System Efficiency R&D Consultant
> > EfficiOS Inc.
> > http://www.efficios.com
> >
> > _______________________________________________
> > lttng-dev mailing list
> > lttng-dev@lists.lttng.org
> > http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
>
> --
> Mathieu Desnoyers
> Operating System Efficiency R&D Consultant
> EfficiOS Inc.
> http://www.efficios.com
>
> _______________________________________________
> lttng-dev mailing list
> lttng-dev@lists.lttng.org
> http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

[-- Attachment #2: full_bt.txt --]
[-- Type: text/plain, Size: 8370 bytes --]

(gdb) thread apply all bt full
[New Thread 8029]
[New Thread 8050]

Thread 3 (Thread 8050):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S:144
No locals.
#1  0xb762d2e8 in _L_lock_850 () from /localdata/mburton-cn5410-main/build/saos-sim5410/fs/releasefs/debug/ctm10/i486-sim/rootfs/lib/libpthread.so.0
No symbol table info available.
#2  0xb762d12b in __pthread_mutex_lock (mutex=0xb7822e80) at pthread_mutex_lock.c:61
        ignore3 = 0
        ignore1 = -512
        ignore2 = 128
        type = <value optimized out>
        __PRETTY_FUNCTION__ =           "__pthread_mutex_lock"
#3  0xb77283b6 in pthread_mutex_lock (mutex=0xb7822e80) at forward.c:182
        __p = 0xfffffe00
#4  0xb77ffb38 in ust_lock () at ltt-events.c:70
No locals.
#5  0xb77fa05d in ust_listener_thread (arg=0xb781e9a0) at lttng-ust-comm.c:681
        sock_info = 0xb781e9a0
        sock = 0
        ret = 0
        prev_connect_failed = 0
        has_waited = 0
        __func__ =           "ust_listener_thread"
        __PRETTY_FUNCTION__ =           "ust_listener_thread"
#6  0xb762b723 in start_thread (arg=0xb6e21b70) at pthread_create.c:304
        __res = <value optimized out>
        __ignore1 = <value optimized out>
        __ignore2 = <value optimized out>
        pd = 0xb6e21b70
        unwind_buf = {
          cancel_jmp_buf =             {{
              jmp_buf =                 {-1218203660,
                0,
                8195840,
                -1226699416,
                119572838,
                106195301}, 
              mask_was_saved = 0
            }}, 
          priv = {
            pad =               {0x0,
              0x0,
              0x0,
              0x0}, 
            data = {
              prev = 0x0, 
              cleanup = 0x0, 
              canceltype = 0
            }
          }
        }
        not_first_call = 0
        robust = <value optimized out>
---Type <return> to continue, or q <return> to quit---
        freesize = <value optimized out>
        __PRETTY_FUNCTION__ =           "start_thread"
#7  0xb771aade in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:133
No locals.

Thread 2 (Thread 8029):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S:144
No locals.
#1  0xb762d2e8 in _L_lock_850 () from /localdata/mburton-cn5410-main/build/saos-sim5410/fs/releasefs/debug/ctm10/i486-sim/rootfs/lib/libpthread.so.0
No symbol table info available.
#2  0xb762d12b in __pthread_mutex_lock (mutex=0xb7822e80) at pthread_mutex_lock.c:61
        ignore3 = 5
        ignore1 = -512
        ignore2 = 128
        type = <value optimized out>
        __PRETTY_FUNCTION__ =           "__pthread_mutex_lock"
#3  0xb77283b6 in pthread_mutex_lock (mutex=0xb7822e80) at forward.c:182
        __p = 0xfffffe00
#4  0xb77ffb38 in ust_lock () at ltt-events.c:70
No locals.
#5  0xb77fb5aa in ust_before_fork (save_sigset=0xb761fa60) at lttng-ust-comm.c:1021
        all_sigs = {
          __val =             {2147483647,
            4294967294,
            4294967295 <repeats 30 times>}
        }
        ret = 0
        __func__ =           "ust_before_fork"
#6  0xb77ca647 in fork () at ustfork.c:47
        sigset = {
          __val =             {2147221231,
            4294967294,
            3076651693,
            3076651736,
            3078871104,
            3078398428,
            3078785376,
            4294967295,
            448139,
            3078909940,
            3078783968,
            3078784408,
            3076651728,
            3078847288,
            3078784408,
            3076815328,
            1,
            5,
            0,
            5,
            3078556861,
            189,
            3078549504,
---Type <return> to continue, or q <return> to quit---
            194532,
            3078398428,
            3078743412,
            8029,
            3078715683,
            3076653128,
            3078871104,
            2,
            3078399436}
        }
        retval = 2
        plibc_func = 0xb76e32f0 <__libc_fork>
#7  0xb77f91cd in get_wait_shm (sock_info=0xb78209c0, mmap_size=4096) at lttng-ust-comm.c:472
        wait_shm_fd = -1
        ret = -1216417792
        pid = 193988
        __func__ =           "get_wait_shm"
#8  0xb77f99a1 in get_map_shm (sock_info=0xb78209c0) at lttng-ust-comm.c:581
        mmap_size = 4096
        wait_shm_fd = -1218313384
        ret = -1217231946
        wait_shm_mmap = 0xb7822e80 "\002"
        __func__ =           "get_map_shm"
#9  0xb77f9c87 in wait_for_sessiond (sock_info=0xb78209c0) at lttng-ust-comm.c:615
        ret = -1217231866
        __func__ =           "wait_for_sessiond"
#10 0xb77fa02d in ust_listener_thread (arg=0xb78209c0) at lttng-ust-comm.c:668
        sock_info = 0xb78209c0
        sock = 1
        ret = 0
        prev_connect_failed = 1
        has_waited = 0
        __func__ =           "ust_listener_thread"
        __PRETTY_FUNCTION__ =           "ust_listener_thread"
#11 0xb762b723 in start_thread (arg=0xb7621b70) at pthread_create.c:304
        __res = <value optimized out>
        __ignore1 = <value optimized out>
        __ignore2 = <value optimized out>
        pd = 0xb7621b70
        unwind_buf = {
          cancel_jmp_buf =             {{
              jmp_buf =                 {-1218203660,
                0,
                8195840,
                -1218310808,
                119572837,
                106195301}, 
              mask_was_saved = 0
            }}, 
          priv = {
            pad =               {0x0,
              0x0,
              0x0,
---Type <return> to continue, or q <return> to quit---
              0x0}, 
            data = {
              prev = 0x0, 
              cleanup = 0x0, 
              canceltype = 0
            }
          }
        }
        not_first_call = 0
        robust = <value optimized out>
        freesize = <value optimized out>
        __PRETTY_FUNCTION__ =           "start_thread"
#12 0xb771aade in clone () at ../sysdeps/unix/sysv/linux/i386/clone.S:133
No locals.

Thread 1 (Thread 4238):
#0  __lll_lock_wait () at ../nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S:144
No locals.
#1  0xb762d2e8 in _L_lock_850 () from /localdata/mburton-cn5410-main/build/saos-sim5410/fs/releasefs/debug/ctm10/i486-sim/rootfs/lib/libpthread.so.0
No symbol table info available.
#2  0xb762d12b in __pthread_mutex_lock (mutex=0xb7822e80) at pthread_mutex_lock.c:61
        ignore3 = -1218151504
        ignore1 = -512
        ignore2 = 128
        type = <value optimized out>
        __PRETTY_FUNCTION__ =           "__pthread_mutex_lock"
#3  0xb77283b6 in pthread_mutex_lock (mutex=0xb7822e80) at forward.c:182
        __p = 0xfffffe00
#4  0xb77ffb38 in ust_lock () at ltt-events.c:70
No locals.
#5  0xb77fd8cc in ltt_probe_register (desc=0xb77c3e40) at ltt-probes.c:77
        iter = 0xb77f001c
        ret = 0
        i = -1216595880
        __func__ =           "ltt_probe_register"
        __PRETTY_FUNCTION__ =           "ltt_probe_register"
#6  0xb77c210f in __lttng_events_init__ust_libc () at ../include/lttng/ust-tracepoint-event.h:550
        ret = -1216225080
        __PRETTY_FUNCTION__ =           "__lttng_events_init__ust_libc"
#7  0xb77c2412 in __do_global_ctors_aux () from /localdata/mburton-cn5410-main/build/saos-sim5410/fs/releasefs/debug/ctm10/i486-sim/rootfs/usr/lib/liblttng-ust-libc-wrapper.so.0
No locals.
#8  0xb77c1930 in _init () from /localdata/mburton-cn5410-main/build/saos-sim5410/fs/releasefs/debug/ctm10/i486-sim/rootfs/usr/lib/liblttng-ust-libc-wrapper.so.0
No locals.
#9  0xb7838128 in call_init (l=0xb77c4000, argc=1, argv=0xbfa31814, env=0xbfa3181c) at dl-init.c:70
        init_array = <value optimized out>
#10 0xb783823f in _dl_init (main_map=0xb7847908, argc=1, argv=0xbfa31814, env=0xbfa3181c) at dl-init.c:134
        preinit_array = <value optimized out>
        preinit_array_size = <value optimized out>
        i = 3
#11 0xb782984f in _dl_start_user () from /localdata/mburton-cn5410-main/build/saos-sim5410/fs/releasefs/debug/ctm10/i486-sim/rootfs/lib/ld-linux.so.2
No symbol table info available.

[-- Attachment #3: Type: text/plain, Size: 155 bytes --]

_______________________________________________
lttng-dev mailing list
lttng-dev@lists.lttng.org
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

^ permalink raw reply	[flat|nested] 16+ messages in thread
* Re: UST Hanging at Tracepoint
       [not found]           ` <20120612162526.GB16098@Krystal>
  2012-06-12 18:24             ` Burton, Michael
@ 2012-06-12 22:25             ` Burton, Michael
       [not found]             ` <3F56B905CF2083408E8482044F20428AEF7A0ED6@ONWVEXCHMB01.ciena.com>
  2 siblings, 0 replies; 16+ messages in thread
From: Burton, Michael @ 2012-06-12 22:25 UTC (permalink / raw)
  To: Mathieu Desnoyers; +Cc: lttng-dev, dgoulet

Mathieu,

I think there is a deadlock scenario in UST, which has been causing my problem.

sessiond is started as root:
    - creates global sockets ONLY
    - DOES NOT CREATE shm in $HOME/lttng-ust-wait-<uid>

application linked against ust is run as root:
  - in lttng_ust_init constructor
  - ust_listener_thread  (local_apps)
    - fails to connect to local_apps in $HOME/.lttng (as expected)
    - prev_connect_failed=1
    - ust_unlock()
    - restart
    - wait_for_sessiond()
 -->  - ust_lock()
 |      - get_map_shm()
 |        - get_wait_shm()
DEADLOCK    - shm_open() FAILS (not created by sessiond when run by root)
 |            - fork() (trying to create shared memory itself)
 |              - ust_before_fork()
 ------------>    - ust_lock()


You should be able to create this with an empty main, with no tracepoints.  As long as sessiond is started as root so $HOME/lttng-ust-wait-<uid> is not created.  You can also make the lttng-ust constructor (lttng_ust_init) wait forever and then you'll be able to see the deadlock in gdb without even leaving the lttng_ust_init constructor.

Also, a thing that is confusing in the lttng_ust_init constructor.  Why is the local ust listener thread always started, but the global one is only started if local_apps.allowed=1?  Shouldn't that be reversed?  The code would end up like this:

        ret = pthread_create(&global_apps.ust_listener, NULL,
                        ust_listener_thread, &global_apps);

        if (local_apps.allowed) {
                ret = pthread_create(&local_apps.ust_listener, NULL,
                                ust_listener_thread, &local_apps);
        } else {
                handle_register_done(&local_apps);
        }

But this is unrelated to the deadlock scenario.

Michael

-----Original Message-----
From: Mathieu Desnoyers [mailto:mathieu.desnoyers@efficios.com]
Sent: Tuesday, June 12, 2012 12:25 PM
To: Burton, Michael
Cc: dgoulet@efficios.com; lttng-dev@lists.lttng.org
Subject: Re: [lttng-dev] UST Hanging at Tracepoint

Please build you app and lttng-ust with:

make CFLAGS=-g   (so with -O0 optimisation, and debug info)

then, under gdb, please provide the full backtrace of each thread when
stucked with:

thread apply all bt full

Thanks,

Mathieu


* Burton, Michael (mburton@ciena.com) wrote:
> Also, I stepped through the code with gdb and things hang within the UST constructors and the main function of my application is never reached.
>
> -----Original Message-----
> From: Burton, Michael [mailto:mburton@ciena.com]
> Sent: Tuesday, June 12, 2012 11:45 AM
> To: Mathieu Desnoyers; dgoulet@efficios.com
> Cc: lttng-dev@lists.lttng.org
> Subject: Re: [lttng-dev] UST Hanging at Tracepoint
>
> I put a fprintf at the beginning of main() and it never prints.
>
> -----Original Message-----
> From: Mathieu Desnoyers [mailto:mathieu.desnoyers@efficios.com]
> Sent: Tuesday, June 12, 2012 10:14 AM
> To: Burton, Michael; dgoulet@efficios.com
> Cc: lttng-dev@lists.lttng.org
> Subject: Re: [lttng-dev] UST Hanging at Tracepoint
>
> * Mathieu Desnoyers (mathieu.desnoyers@efficios.com) wrote:
> > Hi Michael,
> >
> > Please note that you should be able to run UST with either (or both) of:
> >
> > - root lttng-sessiond, for system-wide tracing
> > - per-user lttng-sessiond, for per-user tracing.
> >
> > It is entirely normal that one of the two UST threads within the
> > application can't find an active sessiond if, for instance, you have
> > just a root (and no per-user) sessiond running.
> >
> > So although I'm glad that you got things working, I fail to understand
> > your explanation: it should have worked fine with the root sessiond,
> > even if no ~/.lttng exists.
> >
> > David, can you look into this ?
>
> hrm, further insight: is your program _really_ hang ? Try to make sure
> you put a fprintf(stderr, ".....") at the beginning of your main().
> Looking once more at your detailed UST output indicates that the
> per-user UST thread tells you that it cannot see any running sessiond.
> So yes, this thread blocks, but the thread talking to the root sessiond
> can very well be working fine, and your application might also be
> running fine. This should be double-checked.
>
> Thanks,
>
> Mathieu
>
> >
> > Thanks,
> >
> > Mathieu
> >
> > * Burton, Michael (mburton@ciena.com) wrote:
> > > Mathieu,
> > >
> > > I figured out the problem.  When we start sessiond on our system as root, the /.lttng/ folder is not created, along with its contents.  UST timed out since there was no sock_path.  I was able to get UST working once I got sessiond running.
> > >
> > > Thanks.
> > > Michael
> > >
> > > ________________________________________
> > > From: Mathieu Desnoyers [mathieu.desnoyers@efficios.com]
> > > Sent: 08 June 2012 21:38
> > > To: Burton, Michael
> > > Cc: lttng-dev@lists.lttng.org
> > > Subject: Re: [lttng-dev] UST Hanging at Tracepoint
> > >
> > > * Burton, Michael (mburton@ciena.com) wrote:
> > > > For further insight, I've attached the strace from the hanging
> > > > program.  I also upgraded LTTng-UST to 2.0.3 and userspace-rcu to
> > > > 0.7.3.
> > >
> > > Haven't had time to look at it yet, but could you also provide the
> > > output of:
> > >
> > > LTTNG_UST_DEBUG=1 youprogname
> > >
> > > (starting your UST instrumented program with LTTNG_UST_DEBUG=1 env. var.
> > > set)
> > >
> > > Thanks!
> > >
> > > Mathieu
> > >
> > > >
> > > > Michael
> > > >
> > > > From: Burton, Michael [mailto:mburton@ciena.com]
> > > > Sent: Thursday, June 07, 2012 2:16 PM
> > > > To: lttng-dev@lists.lttng.org
> > > > Subject: [lttng-dev] UST Hanging at Tracepoint
> > > >
> > > > I am working on getting LTTng 2.0 working in our 2.6.35 kernel.  I have kernel tracing working but I'm having problems with UST.
> > > >
> > > > I have a program with a dynamic UST tracepoint.  When I run the program with all UST events enabled (lttng enable-event -a -u) the program hangs on the tracepoint.  The last line from the UST debug is:
> > > >
> > > > libust[6184/6185]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)
> > > >
> > > > Any insight into why this is happening?  I've attached the UST and lttng-sessiond debug generated by the program.
> > > >
> > > > I am running the following:
> > > > lttng-modules-2.6.32 (found through this mailing list)
> > > > lttng-tools-2.0.1
> > > > lttng-ust-2.0.2
> > > > userspace-rcu-0.7.0
> > > >
> > > > Thanks,
> > > > Michael
> > >
> > > Content-Description: strace_debug.txt
> > > > execve("/ciena/bin/idp", ["idp", "help"], [/* 11 vars */]) = 0
> > > > brk(0)                                  = 0x804e000
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb783c000
> > > > access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
> > > > open("/etc/ld.so.cache", O_RDONLY)      = 3
> > > > fstat64(3, {st_mode=S_IFREG|0644, st_size=17420, ...}) = 0
> > > > mmap2(NULL, 17420, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7837000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220A\0\0004\0\0\0$"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=209876, ...}) = 0
> > > > mmap2(NULL, 212940, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7803000
> > > > mmap2(0xb7832000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2e) = 0xb7832000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust-ctl.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200-\0\0004\0\0\0\274"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=140020, ...}) = 0
> > > > mmap2(NULL, 142828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77e0000
> > > > mmap2(0xb7802000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21) = 0xb7802000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust-fork.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\5\0\0004\0\0\0X"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=3864, ...}) = 0
> > > > mmap2(NULL, 6820, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77de000
> > > > mmap2(0xb77df000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77df000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust-tracepoint.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220\v\0\0004\0\0\0008"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=16632, ...}) = 0
> > > > mmap2(NULL, 19944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d9000
> > > > mmap2(0xb77dd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xb77dd000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liblttng-ust-libc-wrapper.so.0", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\t\0\0004\0\0\0\364"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=9300, ...}) = 0
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77d8000
> > > > mmap2(NULL, 12104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d5000
> > > > mmap2(0xb77d7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77d7000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d0000
> > > > mmap2(0xb77d4000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77d4000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-common.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\5\0\0004\0\0\0\34"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=5084, ...}) = 0
> > > > mmap2(NULL, 8032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ce000
> > > > mmap2(0xb77cf000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77cf000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-qsbr.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\22\0\0004\0\0\0\374"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c9000
> > > > mmap2(0xb77cd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77cd000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-mb.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c4000
> > > > mmap2(0xb77c8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c8000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-signal.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\23\0\0004\0\0\0\10"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18712, ...}) = 0
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77c3000
> > > > mmap2(NULL, 21704, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77bd000
> > > > mmap2(0xb77c2000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c2000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-cds.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\f\0\0004\0\0\0t"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=26204, ...}) = 0
> > > > mmap2(NULL, 25004, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b6000
> > > > mmap2(0xb77bc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb77bc000
> > > > close(3)                                = 0
> > > > open("/usr/lib/liburcu-bp.so.1", O_RDONLY) = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\24\0\0004\0\0\0\360"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=19968, ...}) = 0
> > > > mmap2(NULL, 23128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b0000
> > > > mmap2(0xb77b5000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77b5000
> > > > close(3)                                = 0
> > > > open("/lib/libdl.so.2", O_RDONLY)       = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \n\0\0004\0\0\0<"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=9668, ...}) = 0
> > > > mmap2(NULL, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ac000
> > > > mmap2(0xb77ae000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77ae000
> > > > close(3)                                = 0
> > > > open("/lib/libuuid.so.1", O_RDONLY)     = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@\r\0\0004\0\0\0\350"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=12872, ...}) = 0
> > > > mmap2(NULL, 15632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77a8000
> > > > mmap2(0xb77ab000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2) = 0xb77ab000
> > > > close(3)                                = 0
> > > > open("/lib/libc.so.6", O_RDONLY)        = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220n\1\0004\0\0\0L"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=1347780, ...}) = 0
> > > > mmap2(NULL, 1358120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb765c000
> > > > mprotect(0xb77a1000, 4096, PROT_NONE)   = 0
> > > > mmap2(0xb77a2000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x145) = 0xb77a2000
> > > > mmap2(0xb77a5000, 10536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb77a5000
> > > > close(3)                                = 0
> > > > open("/lib/librt.so.1", O_RDONLY)       = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\30\0\0004\0\0\0\230"..., 512) = 512
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb765b000
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=30616, ...}) = 0
> > > > mmap2(NULL, 33364, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7652000
> > > > mmap2(0xb7659000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7659000
> > > > close(3)                                = 0
> > > > open("/lib/libpthread.so.0", O_RDONLY)  = 3
> > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360J\0\0004\0\0\0\354"..., 512) = 512
> > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=88420, ...}) = 0
> > > > mmap2(NULL, 98828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7639000
> > > > mmap2(0xb764e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0xb764e000
> > > > mmap2(0xb7650000, 4620, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7650000
> > > > close(3)                                = 0
> > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7638000
> > > > mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7636000
> > > > set_thread_area({entry_number:-1 -> 6, base_addr:0xb7636d80, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
> > > > mprotect(0xb764e000, 4096, PROT_READ)   = 0
> > > > mprotect(0xb7659000, 4096, PROT_READ)   = 0
> > > > mprotect(0xb77a2000, 8192, PROT_READ)   = 0
> > > > mprotect(0xb77ae000, 4096, PROT_READ)   = 0
> > > > mprotect(0xb785a000, 4096, PROT_READ)   = 0
> > > > munmap(0xb7837000, 17420)               = 0
> > > > set_tid_address(0xb7636de8)             = 3050
> > > > set_robust_list(0xb7636df0, 0xc)        = 0
> > > > futex(0xbfaaf560, FUTEX_WAKE_PRIVATE, 1) = 0
> > > > futex(0xbfaaf560, 0x189 /* FUTEX_??? */, 1, NULL, bfaaf570) = -1 EAGAIN (Resource temporarily unavailable)
> > > > rt_sigaction(SIGRTMIN, {0xb763d4f0, [], SA_SIGINFO}, NULL, 8) = 0
> > > > rt_sigaction(SIGRT_1, {0xb763d9c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > > > rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> > > > getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
> > > > uname({sys="Linux", node="5410", ...})  = 0
> > > > rt_sigaction(SIGUSR1, {0xb77bec0a, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > > > futex(0xb77af06c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
> > > > brk(0)                                  = 0x804e000
> > > > brk(0x806f000)                          = 0x806f000
> > > > gettimeofday({1339187216, 880428}, NULL) = 0
> > > > open("/dev/urandom", O_RDONLY|O_LARGEFILE) = 3
> > > > fcntl64(3, F_GETFD)                     = 0
> > > > fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
> > > > getuid32()                              = 0
> > > > gettimeofday({1339187216, 889603}, NULL) = 0
> > > > gettimeofday({1339187216, 894471}, NULL) = 0
> > > > read(3, "\275\0051F\215\373\371\202\377kfb/\362\303N"..., 16) = 16
> > > > clock_gettime(CLOCK_REALTIME, {1339187216, 902329405}) = 0
> > > > getuid32()                              = 0
> > > > geteuid32()                             = 0
> > > > rt_sigprocmask(SIG_SETMASK, ~[RTMIN RT_1], [], 8) = 0
> > > > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6e36000
> > > > mprotect(0xb6e36000, 4096, PROT_NONE)   = 0
> > > > clone(child_stack=0xb7634d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb7635bd8, {entry_number:6, base_addr:0xb7635b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb7635bd8) = 3051
> > > > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6636000
> > > > mprotect(0xb6636000, 4096, PROT_NONE)   = 0
> > > > clone(child_stack=0xb6e34d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb6e35bd8, {entry_number:6, base_addr:0xb6e35b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb6e35bd8) = 3052
> > > > rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
> > > > gettimeofday({1339187216, 974869}, NULL) = 0
> > > > futex(0xb7836e30, FUTEX_WAIT_PRIVATE, 0, {2, 927460405}) = -1 ETIMEDOUT (Connection timed out)
> > > > gettid()                                = 3050
> > > > write(2, "libust[3050/3050]: Error: Timed o"..., 107libust[3050/3050]: Error: Timed out waiting for ltt-sessiond (in lttng_ust_init() at lttng-ust-comm.c:912)
> > > > ) = 107
> > > > futex(0xb7836e80, FUTEX_WAIT_PRIVATE, 2, NULL
> > > > _______________________________________________
> > > > lttng-dev mailing list
> > > > lttng-dev@lists.lttng.org
> > > > http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
> > >
> > >
> > > --
> > > Mathieu Desnoyers
> > > Operating System Efficiency R&D Consultant
> > > EfficiOS Inc.
> > > http://www.efficios.com
> >
> > --
> > Mathieu Desnoyers
> > Operating System Efficiency R&D Consultant
> > EfficiOS Inc.
> > http://www.efficios.com
> >
> > _______________________________________________
> > lttng-dev mailing list
> > lttng-dev@lists.lttng.org
> > http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
>
> --
> Mathieu Desnoyers
> Operating System Efficiency R&D Consultant
> EfficiOS Inc.
> http://www.efficios.com
>
> _______________________________________________
> lttng-dev mailing list
> lttng-dev@lists.lttng.org
> http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

--
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

^ permalink raw reply	[flat|nested] 16+ messages in thread
* Re: UST Hanging at Tracepoint
       [not found]             ` <3F56B905CF2083408E8482044F20428AEF7A0ED6@ONWVEXCHMB01.ciena.com>
@ 2012-06-13  0:09               ` Mathieu Desnoyers
       [not found]               ` <20120613000950.GD21966@Krystal>
  1 sibling, 0 replies; 16+ messages in thread
From: Mathieu Desnoyers @ 2012-06-13  0:09 UTC (permalink / raw)
  To: Burton, Michael; +Cc: lttng-dev, dgoulet

* Burton, Michael (mburton@ciena.com) wrote:
> Mathieu,
> 
> I think there is a deadlock scenario in UST, which has been causing my problem.

Good catch !

> 
> sessiond is started as root:
>     - creates global sockets ONLY
>     - DOES NOT CREATE shm in $HOME/lttng-ust-wait-<uid>
> 
> application linked against ust is run as root:
>   - in lttng_ust_init constructor
>   - ust_listener_thread  (local_apps)
>     - fails to connect to local_apps in $HOME/.lttng (as expected)
>     - prev_connect_failed=1
>     - ust_unlock()
>     - restart
>     - wait_for_sessiond()
>  -->  - ust_lock()
>  |      - get_map_shm()
>  |        - get_wait_shm()
> DEADLOCK    - shm_open() FAILS (not created by sessiond when run by root)
>  |            - fork() (trying to create shared memory itself)
>  |              - ust_before_fork()
>  ------------>    - ust_lock()
> 
> 
> You should be able to create this with an empty main, with no
> tracepoints.  As long as sessiond is started as root so
> $HOME/lttng-ust-wait-<uid> is not created.  You can also make the
> lttng-ust constructor (lttng_ust_init) wait forever and then you'll be
> able to see the deadlock in gdb without even leaving the
> lttng_ust_init constructor.

Ah, I see. This deadlock is caused by the interaction between
liblttng-ust-libc-wrapper and liblttng-ust (the fork override is
performed by liblttng-ust-libc-wrapper). I'll have to look into this
issue further. More on that soon.

> Also, a thing that is confusing in the lttng_ust_init constructor.
> Why is the local ust listener thread always started, but the global
> one is only started if local_apps.allowed=1?  Shouldn't that be
> reversed?  The code would end up like this:
> 
>         ret = pthread_create(&global_apps.ust_listener, NULL,
>                         ust_listener_thread, &global_apps);
> 
>         if (local_apps.allowed) {
>                 ret = pthread_create(&local_apps.ust_listener, NULL,
>                                 ust_listener_thread, &local_apps);
>         } else {
>                 handle_register_done(&local_apps);
>         }
> 
> But this is unrelated to the deadlock scenario.

Good catch too! This is a really stupid error on my behalf. Fixing
immediately.

Thanks!

Mathieu

> 
> Michael
> 
> -----Original Message-----
> From: Mathieu Desnoyers [mailto:mathieu.desnoyers@efficios.com]
> Sent: Tuesday, June 12, 2012 12:25 PM
> To: Burton, Michael
> Cc: dgoulet@efficios.com; lttng-dev@lists.lttng.org
> Subject: Re: [lttng-dev] UST Hanging at Tracepoint
> 
> Please build you app and lttng-ust with:
> 
> make CFLAGS=-g   (so with -O0 optimisation, and debug info)
> 
> then, under gdb, please provide the full backtrace of each thread when
> stucked with:
> 
> thread apply all bt full
> 
> Thanks,
> 
> Mathieu
> 
> 
> * Burton, Michael (mburton@ciena.com) wrote:
> > Also, I stepped through the code with gdb and things hang within the UST constructors and the main function of my application is never reached.
> >
> > -----Original Message-----
> > From: Burton, Michael [mailto:mburton@ciena.com]
> > Sent: Tuesday, June 12, 2012 11:45 AM
> > To: Mathieu Desnoyers; dgoulet@efficios.com
> > Cc: lttng-dev@lists.lttng.org
> > Subject: Re: [lttng-dev] UST Hanging at Tracepoint
> >
> > I put a fprintf at the beginning of main() and it never prints.
> >
> > -----Original Message-----
> > From: Mathieu Desnoyers [mailto:mathieu.desnoyers@efficios.com]
> > Sent: Tuesday, June 12, 2012 10:14 AM
> > To: Burton, Michael; dgoulet@efficios.com
> > Cc: lttng-dev@lists.lttng.org
> > Subject: Re: [lttng-dev] UST Hanging at Tracepoint
> >
> > * Mathieu Desnoyers (mathieu.desnoyers@efficios.com) wrote:
> > > Hi Michael,
> > >
> > > Please note that you should be able to run UST with either (or both) of:
> > >
> > > - root lttng-sessiond, for system-wide tracing
> > > - per-user lttng-sessiond, for per-user tracing.
> > >
> > > It is entirely normal that one of the two UST threads within the
> > > application can't find an active sessiond if, for instance, you have
> > > just a root (and no per-user) sessiond running.
> > >
> > > So although I'm glad that you got things working, I fail to understand
> > > your explanation: it should have worked fine with the root sessiond,
> > > even if no ~/.lttng exists.
> > >
> > > David, can you look into this ?
> >
> > hrm, further insight: is your program _really_ hang ? Try to make sure
> > you put a fprintf(stderr, ".....") at the beginning of your main().
> > Looking once more at your detailed UST output indicates that the
> > per-user UST thread tells you that it cannot see any running sessiond.
> > So yes, this thread blocks, but the thread talking to the root sessiond
> > can very well be working fine, and your application might also be
> > running fine. This should be double-checked.
> >
> > Thanks,
> >
> > Mathieu
> >
> > >
> > > Thanks,
> > >
> > > Mathieu
> > >
> > > * Burton, Michael (mburton@ciena.com) wrote:
> > > > Mathieu,
> > > >
> > > > I figured out the problem.  When we start sessiond on our system as root, the /.lttng/ folder is not created, along with its contents.  UST timed out since there was no sock_path.  I was able to get UST working once I got sessiond running.
> > > >
> > > > Thanks.
> > > > Michael
> > > >
> > > > ________________________________________
> > > > From: Mathieu Desnoyers [mathieu.desnoyers@efficios.com]
> > > > Sent: 08 June 2012 21:38
> > > > To: Burton, Michael
> > > > Cc: lttng-dev@lists.lttng.org
> > > > Subject: Re: [lttng-dev] UST Hanging at Tracepoint
> > > >
> > > > * Burton, Michael (mburton@ciena.com) wrote:
> > > > > For further insight, I've attached the strace from the hanging
> > > > > program.  I also upgraded LTTng-UST to 2.0.3 and userspace-rcu to
> > > > > 0.7.3.
> > > >
> > > > Haven't had time to look at it yet, but could you also provide the
> > > > output of:
> > > >
> > > > LTTNG_UST_DEBUG=1 youprogname
> > > >
> > > > (starting your UST instrumented program with LTTNG_UST_DEBUG=1 env. var.
> > > > set)
> > > >
> > > > Thanks!
> > > >
> > > > Mathieu
> > > >
> > > > >
> > > > > Michael
> > > > >
> > > > > From: Burton, Michael [mailto:mburton@ciena.com]
> > > > > Sent: Thursday, June 07, 2012 2:16 PM
> > > > > To: lttng-dev@lists.lttng.org
> > > > > Subject: [lttng-dev] UST Hanging at Tracepoint
> > > > >
> > > > > I am working on getting LTTng 2.0 working in our 2.6.35 kernel.  I have kernel tracing working but I'm having problems with UST.
> > > > >
> > > > > I have a program with a dynamic UST tracepoint.  When I run the program with all UST events enabled (lttng enable-event -a -u) the program hangs on the tracepoint.  The last line from the UST debug is:
> > > > >
> > > > > libust[6184/6185]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)
> > > > >
> > > > > Any insight into why this is happening?  I've attached the UST and lttng-sessiond debug generated by the program.
> > > > >
> > > > > I am running the following:
> > > > > lttng-modules-2.6.32 (found through this mailing list)
> > > > > lttng-tools-2.0.1
> > > > > lttng-ust-2.0.2
> > > > > userspace-rcu-0.7.0
> > > > >
> > > > > Thanks,
> > > > > Michael
> > > >
> > > > Content-Description: strace_debug.txt
> > > > > execve("/ciena/bin/idp", ["idp", "help"], [/* 11 vars */]) = 0
> > > > > brk(0)                                  = 0x804e000
> > > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb783c000
> > > > > access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
> > > > > open("/etc/ld.so.cache", O_RDONLY)      = 3
> > > > > fstat64(3, {st_mode=S_IFREG|0644, st_size=17420, ...}) = 0
> > > > > mmap2(NULL, 17420, PROT_READ, MAP_PRIVATE, 3, 0) = 0xb7837000
> > > > > close(3)                                = 0
> > > > > open("/usr/lib/liblttng-ust.so.0", O_RDONLY) = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220A\0\0004\0\0\0$"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=209876, ...}) = 0
> > > > > mmap2(NULL, 212940, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7803000
> > > > > mmap2(0xb7832000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2e) = 0xb7832000
> > > > > close(3)                                = 0
> > > > > open("/usr/lib/liblttng-ust-ctl.so.0", O_RDONLY) = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\200-\0\0004\0\0\0\274"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=140020, ...}) = 0
> > > > > mmap2(NULL, 142828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77e0000
> > > > > mmap2(0xb7802000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21) = 0xb7802000
> > > > > close(3)                                = 0
> > > > > open("/usr/lib/liblttng-ust-fork.so.0", O_RDONLY) = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\0\5\0\0004\0\0\0X"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=3864, ...}) = 0
> > > > > mmap2(NULL, 6820, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77de000
> > > > > mmap2(0xb77df000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77df000
> > > > > close(3)                                = 0
> > > > > open("/usr/lib/liblttng-ust-tracepoint.so.0", O_RDONLY) = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220\v\0\0004\0\0\0008"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=16632, ...}) = 0
> > > > > mmap2(NULL, 19944, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d9000
> > > > > mmap2(0xb77dd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3) = 0xb77dd000
> > > > > close(3)                                = 0
> > > > > open("/usr/lib/liblttng-ust-libc-wrapper.so.0", O_RDONLY) = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\t\0\0004\0\0\0\364"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=9300, ...}) = 0
> > > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77d8000
> > > > > mmap2(NULL, 12104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d5000
> > > > > mmap2(0xb77d7000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77d7000
> > > > > close(3)                                = 0
> > > > > open("/usr/lib/liburcu.so.1", O_RDONLY) = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77d0000
> > > > > mmap2(0xb77d4000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77d4000
> > > > > close(3)                                = 0
> > > > > open("/usr/lib/liburcu-common.so.1", O_RDONLY) = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\340\5\0\0004\0\0\0\34"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=5084, ...}) = 0
> > > > > mmap2(NULL, 8032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ce000
> > > > > mmap2(0xb77cf000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0) = 0xb77cf000
> > > > > close(3)                                = 0
> > > > > open("/usr/lib/liburcu-qsbr.so.1", O_RDONLY) = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360\22\0\0004\0\0\0\374"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c9000
> > > > > mmap2(0xb77cd000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77cd000
> > > > > close(3)                                = 0
> > > > > open("/usr/lib/liburcu-mb.so.1", O_RDONLY) = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\22\0\0004\0\0\0\374"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18188, ...}) = 0
> > > > > mmap2(NULL, 17080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77c4000
> > > > > mmap2(0xb77c8000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c8000
> > > > > close(3)                                = 0
> > > > > open("/usr/lib/liburcu-signal.so.1", O_RDONLY) = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0`\23\0\0004\0\0\0\10"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=18712, ...}) = 0
> > > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb77c3000
> > > > > mmap2(NULL, 21704, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77bd000
> > > > > mmap2(0xb77c2000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77c2000
> > > > > close(3)                                = 0
> > > > > open("/usr/lib/liburcu-cds.so.1", O_RDONLY) = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\300\f\0\0004\0\0\0t"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=26204, ...}) = 0
> > > > > mmap2(NULL, 25004, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b6000
> > > > > mmap2(0xb77bc000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb77bc000
> > > > > close(3)                                = 0
> > > > > open("/usr/lib/liburcu-bp.so.1", O_RDONLY) = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\320\24\0\0004\0\0\0\360"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=19968, ...}) = 0
> > > > > mmap2(NULL, 23128, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77b0000
> > > > > mmap2(0xb77b5000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4) = 0xb77b5000
> > > > > close(3)                                = 0
> > > > > open("/lib/libdl.so.2", O_RDONLY)       = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0 \n\0\0004\0\0\0<"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=9668, ...}) = 0
> > > > > mmap2(NULL, 12408, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77ac000
> > > > > mmap2(0xb77ae000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1) = 0xb77ae000
> > > > > close(3)                                = 0
> > > > > open("/lib/libuuid.so.1", O_RDONLY)     = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0@\r\0\0004\0\0\0\350"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=12872, ...}) = 0
> > > > > mmap2(NULL, 15632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb77a8000
> > > > > mmap2(0xb77ab000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2) = 0xb77ab000
> > > > > close(3)                                = 0
> > > > > open("/lib/libc.so.6", O_RDONLY)        = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\220n\1\0004\0\0\0L"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=1347780, ...}) = 0
> > > > > mmap2(NULL, 1358120, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb765c000
> > > > > mprotect(0xb77a1000, 4096, PROT_NONE)   = 0
> > > > > mmap2(0xb77a2000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x145) = 0xb77a2000
> > > > > mmap2(0xb77a5000, 10536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb77a5000
> > > > > close(3)                                = 0
> > > > > open("/lib/librt.so.1", O_RDONLY)       = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0p\30\0\0004\0\0\0\230"..., 512) = 512
> > > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb765b000
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=30616, ...}) = 0
> > > > > mmap2(NULL, 33364, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7652000
> > > > > mmap2(0xb7659000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6) = 0xb7659000
> > > > > close(3)                                = 0
> > > > > open("/lib/libpthread.so.0", O_RDONLY)  = 3
> > > > > read(3, "\177ELF\1\1\1\0\0\0\0\0\0\0\0\0\3\0\3\0\1\0\0\0\360J\0\0004\0\0\0\354"..., 512) = 512
> > > > > fstat64(3, {st_mode=S_IFREG|0755, st_size=88420, ...}) = 0
> > > > > mmap2(NULL, 98828, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0xb7639000
> > > > > mmap2(0xb764e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14) = 0xb764e000
> > > > > mmap2(0xb7650000, 4620, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0xb7650000
> > > > > close(3)                                = 0
> > > > > mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7638000
> > > > > mmap2(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xb7636000
> > > > > set_thread_area({entry_number:-1 -> 6, base_addr:0xb7636d80, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}) = 0
> > > > > mprotect(0xb764e000, 4096, PROT_READ)   = 0
> > > > > mprotect(0xb7659000, 4096, PROT_READ)   = 0
> > > > > mprotect(0xb77a2000, 8192, PROT_READ)   = 0
> > > > > mprotect(0xb77ae000, 4096, PROT_READ)   = 0
> > > > > mprotect(0xb785a000, 4096, PROT_READ)   = 0
> > > > > munmap(0xb7837000, 17420)               = 0
> > > > > set_tid_address(0xb7636de8)             = 3050
> > > > > set_robust_list(0xb7636df0, 0xc)        = 0
> > > > > futex(0xbfaaf560, FUTEX_WAKE_PRIVATE, 1) = 0
> > > > > futex(0xbfaaf560, 0x189 /* FUTEX_??? */, 1, NULL, bfaaf570) = -1 EAGAIN (Resource temporarily unavailable)
> > > > > rt_sigaction(SIGRTMIN, {0xb763d4f0, [], SA_SIGINFO}, NULL, 8) = 0
> > > > > rt_sigaction(SIGRT_1, {0xb763d9c0, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > > > > rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
> > > > > getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
> > > > > uname({sys="Linux", node="5410", ...})  = 0
> > > > > rt_sigaction(SIGUSR1, {0xb77bec0a, [], SA_RESTART|SA_SIGINFO}, NULL, 8) = 0
> > > > > futex(0xb77af06c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
> > > > > brk(0)                                  = 0x804e000
> > > > > brk(0x806f000)                          = 0x806f000
> > > > > gettimeofday({1339187216, 880428}, NULL) = 0
> > > > > open("/dev/urandom", O_RDONLY|O_LARGEFILE) = 3
> > > > > fcntl64(3, F_GETFD)                     = 0
> > > > > fcntl64(3, F_SETFD, FD_CLOEXEC)         = 0
> > > > > getuid32()                              = 0
> > > > > gettimeofday({1339187216, 889603}, NULL) = 0
> > > > > gettimeofday({1339187216, 894471}, NULL) = 0
> > > > > read(3, "\275\0051F\215\373\371\202\377kfb/\362\303N"..., 16) = 16
> > > > > clock_gettime(CLOCK_REALTIME, {1339187216, 902329405}) = 0
> > > > > getuid32()                              = 0
> > > > > geteuid32()                             = 0
> > > > > rt_sigprocmask(SIG_SETMASK, ~[RTMIN RT_1], [], 8) = 0
> > > > > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6e36000
> > > > > mprotect(0xb6e36000, 4096, PROT_NONE)   = 0
> > > > > clone(child_stack=0xb7634d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb7635bd8, {entry_number:6, base_addr:0xb7635b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb7635bd8) = 3051
> > > > > mmap2(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0xb6636000
> > > > > mprotect(0xb6636000, 4096, PROT_NONE)   = 0
> > > > > clone(child_stack=0xb6e34d64, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID|CLONE_DETACHED, parent_tidptr=0xb6e35bd8, {entry_number:6, base_addr:0xb6e35b70, limit:1048575, seg_32bit:1, contents:0, read_exec_only:0, limit_in_pages:1, seg_not_present:0, useable:1}, child_tidptr=0xb6e35bd8) = 3052
> > > > > rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
> > > > > gettimeofday({1339187216, 974869}, NULL) = 0
> > > > > futex(0xb7836e30, FUTEX_WAIT_PRIVATE, 0, {2, 927460405}) = -1 ETIMEDOUT (Connection timed out)
> > > > > gettid()                                = 3050
> > > > > write(2, "libust[3050/3050]: Error: Timed o"..., 107libust[3050/3050]: Error: Timed out waiting for ltt-sessiond (in lttng_ust_init() at lttng-ust-comm.c:912)
> > > > > ) = 107
> > > > > futex(0xb7836e80, FUTEX_WAIT_PRIVATE, 2, NULL
> > > > > _______________________________________________
> > > > > lttng-dev mailing list
> > > > > lttng-dev@lists.lttng.org
> > > > > http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
> > > >
> > > >
> > > > --
> > > > Mathieu Desnoyers
> > > > Operating System Efficiency R&D Consultant
> > > > EfficiOS Inc.
> > > > http://www.efficios.com
> > >
> > > --
> > > Mathieu Desnoyers
> > > Operating System Efficiency R&D Consultant
> > > EfficiOS Inc.
> > > http://www.efficios.com
> > >
> > > _______________________________________________
> > > lttng-dev mailing list
> > > lttng-dev@lists.lttng.org
> > > http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
> >
> > --
> > Mathieu Desnoyers
> > Operating System Efficiency R&D Consultant
> > EfficiOS Inc.
> > http://www.efficios.com
> >
> > _______________________________________________
> > lttng-dev mailing list
> > lttng-dev@lists.lttng.org
> > http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
> 
> --
> Mathieu Desnoyers
> Operating System Efficiency R&D Consultant
> EfficiOS Inc.
> http://www.efficios.com

-- 
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

^ permalink raw reply	[flat|nested] 16+ messages in thread
* Re: UST Hanging at Tracepoint
       [not found]               ` <20120613000950.GD21966@Krystal>
@ 2012-06-13  0:21                 ` Mathieu Desnoyers
       [not found]                 ` <20120613002128.GE21966@Krystal>
  1 sibling, 0 replies; 16+ messages in thread
From: Mathieu Desnoyers @ 2012-06-13  0:21 UTC (permalink / raw)
  To: Burton, Michael; +Cc: lttng-dev, dgoulet

* Mathieu Desnoyers (mathieu.desnoyers@efficios.com) wrote:
> * Burton, Michael (mburton@ciena.com) wrote:
> > Mathieu,
> > 
> > I think there is a deadlock scenario in UST, which has been causing my problem.
> 
> Good catch !
> 
> > 
> > sessiond is started as root:
> >     - creates global sockets ONLY
> >     - DOES NOT CREATE shm in $HOME/lttng-ust-wait-<uid>
> > 
> > application linked against ust is run as root:
> >   - in lttng_ust_init constructor
> >   - ust_listener_thread  (local_apps)
> >     - fails to connect to local_apps in $HOME/.lttng (as expected)
> >     - prev_connect_failed=1
> >     - ust_unlock()
> >     - restart
> >     - wait_for_sessiond()
> >  -->  - ust_lock()
> >  |      - get_map_shm()
> >  |        - get_wait_shm()
> > DEADLOCK    - shm_open() FAILS (not created by sessiond when run by root)
> >  |            - fork() (trying to create shared memory itself)
> >  |              - ust_before_fork()
> >  ------------>    - ust_lock()
> > 
> > 
> > You should be able to create this with an empty main, with no
> > tracepoints.  As long as sessiond is started as root so
> > $HOME/lttng-ust-wait-<uid> is not created.  You can also make the
> > lttng-ust constructor (lttng_ust_init) wait forever and then you'll be
> > able to see the deadlock in gdb without even leaving the
> > lttng_ust_init constructor.
> 
> Ah, I see. This deadlock is caused by the interaction between
> liblttng-ust-libc-wrapper and liblttng-ust (the fork override is
> performed by liblttng-ust-libc-wrapper). I'll have to look into this
> issue further. More on that soon.

Actually, I meant that it comes from interaction between
liblttng-ust-fork and liblttng-ust (sorry about the confusion above).
I'll see if I can provide a direct access to the fork() symbol to
liblttng-ust, using a weak symbol so it is defined even when
liblttng-ust-fork is not linked.

More on that soon...

Mathieu

-- 
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

^ permalink raw reply	[flat|nested] 16+ messages in thread
* Re: UST Hanging at Tracepoint
       [not found]                 ` <20120613002128.GE21966@Krystal>
@ 2012-06-13  8:29                   ` Mathieu Desnoyers
  0 siblings, 0 replies; 16+ messages in thread
From: Mathieu Desnoyers @ 2012-06-13  8:29 UTC (permalink / raw)
  To: Burton, Michael; +Cc: lttng-dev, dgoulet

* Mathieu Desnoyers (mathieu.desnoyers@efficios.com) wrote:
> * Mathieu Desnoyers (mathieu.desnoyers@efficios.com) wrote:
> > * Burton, Michael (mburton@ciena.com) wrote:
> > > Mathieu,
> > > 
> > > I think there is a deadlock scenario in UST, which has been causing my problem.
> > 
> > Good catch !
> > 
> > > 
> > > sessiond is started as root:
> > >     - creates global sockets ONLY
> > >     - DOES NOT CREATE shm in $HOME/lttng-ust-wait-<uid>
> > > 
> > > application linked against ust is run as root:
> > >   - in lttng_ust_init constructor
> > >   - ust_listener_thread  (local_apps)
> > >     - fails to connect to local_apps in $HOME/.lttng (as expected)
> > >     - prev_connect_failed=1
> > >     - ust_unlock()
> > >     - restart
> > >     - wait_for_sessiond()
> > >  -->  - ust_lock()
> > >  |      - get_map_shm()
> > >  |        - get_wait_shm()
> > > DEADLOCK    - shm_open() FAILS (not created by sessiond when run by root)
> > >  |            - fork() (trying to create shared memory itself)
> > >  |              - ust_before_fork()
> > >  ------------>    - ust_lock()
> > > 
> > > 
> > > You should be able to create this with an empty main, with no
> > > tracepoints.  As long as sessiond is started as root so
> > > $HOME/lttng-ust-wait-<uid> is not created.  You can also make the
> > > lttng-ust constructor (lttng_ust_init) wait forever and then you'll be
> > > able to see the deadlock in gdb without even leaving the
> > > lttng_ust_init constructor.
> > 
> > Ah, I see. This deadlock is caused by the interaction between
> > liblttng-ust-libc-wrapper and liblttng-ust (the fork override is
> > performed by liblttng-ust-libc-wrapper). I'll have to look into this
> > issue further. More on that soon.
> 
> Actually, I meant that it comes from interaction between
> liblttng-ust-fork and liblttng-ust (sorry about the confusion above).
> I'll see if I can provide a direct access to the fork() symbol to
> liblttng-ust, using a weak symbol so it is defined even when
> liblttng-ust-fork is not linked.
> 
> More on that soon...

Fixed by master commit:

commit e8508a495280a1396a0319a157409a7ea61ffc5a
Author: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Date:   Wed Jun 13 04:26:34 2012 -0400

    Fix: liblttng-ust-fork deadlock

and merged into stable-2.0.

Thanks for pointing out this important bug.

Mathieu

-- 
Mathieu Desnoyers
Operating System Efficiency R&D Consultant
EfficiOS Inc.
http://www.efficios.com

^ permalink raw reply	[flat|nested] 16+ messages in thread
* UST Hanging at Tracepoint
@ 2012-06-07 18:16 Burton, Michael
  0 siblings, 0 replies; 16+ messages in thread
From: Burton, Michael @ 2012-06-07 18:16 UTC (permalink / raw)
  To: lttng-dev


[-- Attachment #1.1: Type: text/plain, Size: 751 bytes --]

I am working on getting LTTng 2.0 working in our 2.6.35 kernel.  I have kernel tracing working but I'm having problems with UST.

I have a program with a dynamic UST tracepoint.  When I run the program with all UST events enabled (lttng enable-event -a -u) the program hangs on the tracepoint.  The last line from the UST debug is:

libust[6184/6185]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)

Any insight into why this is happening?  I've attached the UST and lttng-sessiond debug generated by the program.

I am running the following:
lttng-modules-2.6.32 (found through this mailing list)
lttng-tools-2.0.1
lttng-ust-2.0.2
userspace-rcu-0.7.0

Thanks,
Michael

[-- Attachment #1.2: Type: text/html, Size: 5582 bytes --]

[-- Attachment #2: ust_debug.txt --]
[-- Type: text/plain, Size: 3112 bytes --]

liblttng_ust_tracepoint[5133/5133]: just registered a tracepoints section from 0xb76fbd44 and having 1 tracepoints (in tracepoint_register_lib() at tracepoint.c:638)
liblttng_ust_tracepoint[5133/5133]: registered tracepoint: lttng_ust:metadata (in tracepoint_register_lib() at tracepoint.c:643)
libust[5133/5133]: LTT : ltt ring buffer client init
 (in ltt_ring_buffer_metadata_client_init() at ltt-ring-buffer-metadata-client.h:334)
libust[5133/5133]: LTT : ltt ring buffer client init
 (in ltt_ring_buffer_client_overwrite_init() at ltt-ring-buffer-client.h:584)
libust[5133/5133]: LTT : ltt ring buffer client init
 (in ltt_ring_buffer_client_discard_init() at ltt-ring-buffer-client.h:584)
libust[5133/5135]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[5133/5135]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[5133/5135]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[5133/5135]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[5133/5135]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[5133/5135]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[5133/5135]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[5133/5135]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[5133/5135]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
liblttng_ust_tracepoint[5133/5135]: Registering probe to tracepoint lttng_ust:metadata (in __tracepoint_probe_register() at tracepoint.c:426)
libust[5133/5135]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[5133/5135]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[5133/5135]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[5133/5135]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[5133/5135]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[5133/5135]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[5133/5135]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[5133/5135]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[5133/5135]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[5133/5135]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[5133/5135]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[5133/5135]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[5133/5135]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[5133/5135]: message received
 (in ust_listener_thread() at lttng-ust-comm.c:766)
libust[5133/5135]: message successfully sent (in send_reply() at lttng-ust-comm.c:202)
libust[5133/5134]: Info: sessiond not accepting connections to local apps socket (in ust_listener_thread() at lttng-ust-comm.c:699)

[-- Attachment #3: sessiond_debug.txt --]
[-- Type: text/plain, Size: 4616 bytes --]

DEBUG1: UST registration received with pid:6184 ppid:2158 uid:0 gid:0 sock:19 name:idp (version 2.0) [in thread_registration_apps() at main.c:1510]
DEBUG1: Futex n to 1 wake done [in futex_nto1_wake() at futex.c:100]
DEBUG1: Accepting application registration [in thread_registration_apps() at main.c:1423]
DEBUG1: Futex n to 1 wait done [in futex_nto1_wait() at futex.c:87]
DEBUG1: Futex n to 1 prepare done [in futex_nto1_prepare() at futex.c:73]
DEBUG1: Dispatching UST registration pid:6184 ppid:2158 uid:0 gid:0 sock:19 name:idp (version 2.0) [in thread_dispatch_ust_registration() at main.c:1346]
DEBUG1: Woken up but nothing in the UST command queue [in thread_dispatch_ust_registration() at main.c:1334]
DEBUG3: Created hashtable size 4 at 0x80807a0 of type 1 [in lttng_ht_new() at hashtable.c:96]
DEBUG1: App registered with pid:6184 ppid:2158 uid:0 gid:0 sock:19 name:idp (version 2.0) [in ust_app_register() at ust-app.c:1365]
DEBUG2: UST app PID 6184 is compatible with major version 2 (supporting <= 2) [in ust_app_validate_version() at ust-app.c:2625]
DEBUG2: UST app global update for app sock 19 for session id 1 [in ust_app_global_update() at ust-app.c:2295]
DEBUG2: UST app pid: 6184 session id 1 not found, creating it [in create_ust_app_session() at ust-app.c:883]
DEBUG3: Created hashtable size 4 at 0x807f1b8 of type 0 [in lttng_ht_new() at hashtable.c:96]
DEBUG2: Shadow copy of session handle -1 [in shadow_copy_session() at ust-app.c:794]
DEBUG2: Channel channel0 not found on shadow session copy, creating it [in shadow_copy_session() at ust-app.c:823]
DEBUG3: Created hashtable size 4 at 0x80802f0 of type 1 [in lttng_ht_new() at hashtable.c:96]
DEBUG3: Created hashtable size 4 at 0x8080680 of type 0 [in lttng_ht_new() at hashtable.c:96]
DEBUG3: UST app channel channel0 allocated [in alloc_ust_app_channel() at ust-app.c:298]
DEBUG2: UST app shadow copy of channel channel0 started [in shadow_copy_channel() at ust-app.c:734]
DEBUG2: UST event * not found on shadow copy channel [in shadow_copy_channel() at ust-app.c:761]
DEBUG3: Created hashtable size 4 at 0x8080be8 of type 1 [in lttng_ht_new() at hashtable.c:96]
DEBUG3: UST app event * allocated [in alloc_ust_app_event() at ust-app.c:333]
DEBUG3: UST app shadow copy of channel channel0 done [in shadow_copy_channel() at ust-app.c:771]
DEBUG2: UST app session created successfully with handle 1 [in create_ust_app_session() at ust-app.c:907]
DEBUG2: UST app channel channel0 created successfully for pid:6184 and sock:19 [in create_ust_channel() at ust-app.c:621]
DEBUG2: UST app event * created successfully for pid:6184 [in create_ust_event() at ust-app.c:660]
DEBUG1: Starting tracing for ust app pid 6184 [in ust_app_start_trace() at ust-app.c:1970]
DEBUG2: UST metadata opened for app pid 6184 [in create_ust_app_metadata() at ust-app.c:1218]
DEBUG3: mkdir() //lttng-traces/auto-20120607-175715/ust/idp-6184-20120607-180232 with mode 504 for uid 0 and gid 0 [in run_as_mkdir() at runas.c:349]
DEBUG1: Using run_as_clone [in run_as() at runas.c:322]
DEBUG2: UST metadata stream object created for app pid 6184 [in create_ust_app_metadata() at ust-app.c:1243]
DEBUG2: UST stream 1 ready at //lttng-traces/auto-20120607-175715/ust/idp-6184-20120607-180232/channel0_0 [in ust_app_start_trace() at ust-app.c:2039]
DEBUG2: UST stream 2 ready at //lttng-traces/auto-20120607-175715/ust/idp-6184-20120607-180232/channel0_1 [in ust_app_start_trace() at ust-app.c:2039]
DEBUG1: Sending metadata stream fd [in ust_consumer_send_session() at ust-consumer.c:126]
DEBUG1: Sending metadata channel 94 to consumer [in ust_consumer_send_session() at ust-consumer.c:142]
DEBUG1: Sending metadata stream 96 to consumer [in ust_consumer_send_session() at ust-consumer.c:166]
DEBUG1: Sending streams of channel channel0 to UST consumer [in send_channel_streams() at ust-consumer.c:43]
DEBUG1: Sending channel 92 to consumer [in send_channel_streams() at ust-consumer.c:58]
DEBUG1: Sending stream 98 to consumer [in send_channel_streams() at ust-consumer.c:91]
DEBUG1: Sending stream 100 to consumer [in send_channel_streams() at ust-consumer.c:91]
DEBUG1: consumer channel streams sent [in send_channel_streams() at ust-consumer.c:107]
DEBUG1: consumer fds (metadata and channel streams) sent [in ust_consumer_send_session() at ust-consumer.c:201]
DEBUG2: UST trace started for app pid 6184 [in ust_app_global_update() at ust-app.c:2364]
DEBUG1: Apps with sock 19 added to poll set [in thread_manage_apps() at main.c:1280]
DEBUG1: Apps thread polling on 3 fds [in thread_manage_apps() at main.c:1200]

[-- Attachment #4: Type: text/plain, Size: 155 bytes --]

_______________________________________________
lttng-dev mailing list
lttng-dev@lists.lttng.org
http://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev

^ permalink raw reply	[flat|nested] 16+ messages in thread
end of thread, other threads:[~2012-06-13  8:29 UTC | newest]

Thread overview: 16+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
     [not found] <3F56B905CF2083408E8482044F20428AEF604D22@ONWVEXCHMB01.ciena.com>
2012-06-08 20:29 ` UST Hanging at Tracepoint Burton, Michael
     [not found] ` <3F56B905CF2083408E8482044F20428AEF6A2DBB@ONWVEXCHMB01.ciena.com>
2012-06-09  1:38   ` Mathieu Desnoyers
     [not found]   ` <20120609013854.GB10978@Krystal>
2012-06-11 15:08     ` Burton, Michael
2012-06-12  2:48     ` Burton, Michael
     [not found] ` <3F56B905CF2083408E8482044F20428AEF78BC79@ONWVEXCHMB01.ciena.com>
2012-06-12 14:09   ` Mathieu Desnoyers
     [not found]   ` <20120612140925.GB13917@Krystal>
2012-06-12 14:13     ` Mathieu Desnoyers
     [not found]     ` <20120612141344.GC13917@Krystal>
2012-06-12 15:44       ` Burton, Michael
     [not found]       ` <3F56B905CF2083408E8482044F20428AEF7A0AFE@ONWVEXCHMB01.ciena.com>
2012-06-12 16:09         ` Burton, Michael
     [not found]         ` <3F56B905CF2083408E8482044F20428AEF7A0B4C@ONWVEXCHMB01.ciena.com>
2012-06-12 16:25           ` Mathieu Desnoyers
     [not found]           ` <20120612162526.GB16098@Krystal>
2012-06-12 18:24             ` Burton, Michael
2012-06-12 22:25             ` Burton, Michael
     [not found]             ` <3F56B905CF2083408E8482044F20428AEF7A0ED6@ONWVEXCHMB01.ciena.com>
2012-06-13  0:09               ` Mathieu Desnoyers
     [not found]               ` <20120613000950.GD21966@Krystal>
2012-06-13  0:21                 ` Mathieu Desnoyers
     [not found]                 ` <20120613002128.GE21966@Krystal>
2012-06-13  8:29                   ` Mathieu Desnoyers
2012-06-12 15:51     ` David Goulet
2012-06-07 18:16 Burton, Michael

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.