[linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[Abhishek Agarwal]

[-- Attachment #1.1: Type: text/plain, Size: 5602 bytes --]

When a kubernetes pod is scheduled on the node having lvm2 libraries
already installed and trying to run lvm commands using those node binaries
from inside the pod container, the commands hang and are waiting on
something to complete. Although when ctrl+c is pressed the terminal session
resumes and checking the final code for the execution returns a "0" error
code and the commands operation is also carried out successfully.


Below is the command output and strace for the command:-

strace of lvcreate on the pod container scheduled on the node where lvm2
binaries are present:
# strace /sbin/lvm-eg/lvcreate -n test-lv -L 1G shared-vg
execve("/sbin/lvm-eg/lvcreate", ["/sbin/lvm-eg/lvcreate", "-n", "test-lv",
"-L", "1G", "shared-vg"], 0x7ffe7d7a6c98 /* 37 vars */) = 0
brk(NULL)                = 0x55d196f41000
arch_prctl(0x3001 /* ARCH_??? */, 0x7ffdd8247a80) = -1 EINVAL (Invalid
argument)
access("/etc/ld.so.preload", R_OK)   = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=8679, ...}) = 0
mmap(NULL, 8679, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f0d37895000
close(3)                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3,
"\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360A\2\0\0\0\0\0"..., 832)
= 832
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"...,
784, 64) = 784
pread64(3,
"\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32,
848) = 32
pread64(3,
"\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\237\333t\347\262\27\320l\223\27*\202C\370T\177"...,
68, 880) = 68
fstat(3, {st_mode=S_IFREG|0755, st_size=2029560, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x7f0d37893000
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"...,
784, 64) = 784
pread64(3,
"\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32,
848) = 32
pread64(3,
"\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\237\333t\347\262\27\320l\223\27*\202C\370T\177"...,
68, 880) = 68
mmap(NULL, 2037344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
0x7f0d376a1000
mmap(0x7f0d376c3000, 1540096, PROT_READ|PROT_EXEC,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7f0d376c3000
mmap(0x7f0d3783b000, 319488, PROT_READ,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19a000) = 0x7f0d3783b000
mmap(0x7f0d37889000, 24576, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7f0d37889000
mmap(0x7f0d3788f000, 13920, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0d3788f000
close(3)                = 0
arch_prctl(ARCH_SET_FS, 0x7f0d37894580) = 0
mprotect(0x7f0d37889000, 16384, PROT_READ) = 0
mprotect(0x55d196662000, 8192, PROT_READ) = 0
mprotect(0x7f0d378c5000, 4096, PROT_READ) = 0
munmap(0x7f0d37895000, 8679)      = 0
getuid()                = 0
getgid()                = 0
getpid()                = 19163
rt_sigaction(SIGCHLD, {sa_handler=0x55d196657c30, sa_mask=~[RTMIN RT_1],
sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
geteuid()                = 0
brk(NULL)                = 0x55d196f41000
brk(0x55d196f62000)           = 0x55d196f62000
getppid()                = 19160
stat("/", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
stat(".", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
openat(AT_FDCWD, "/sbin/lvm-eg/lvcreate", O_RDONLY) = 3
fcntl(3, F_DUPFD, 10)          = 10
close(3)                = 0
fcntl(10, F_SETFD, FD_CLOEXEC)     = 0
geteuid()                = 0
getegid()                = 0
rt_sigaction(SIGINT, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8)
= 0
rt_sigaction(SIGINT, {sa_handler=0x55d196657c30, sa_mask=~[RTMIN RT_1],
sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
rt_sigaction(SIGQUIT, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0},
8) = 0
rt_sigaction(SIGQUIT, {sa_handler=SIG_DFL, sa_mask=~[RTMIN RT_1],
sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
rt_sigaction(SIGTERM, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0},
8) = 0
rt_sigaction(SIGTERM, {sa_handler=SIG_DFL, sa_mask=~[RTMIN RT_1],
sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
read(10, "#!/bin/sh\npath=$(/sbin/lvm-eg/ge"..., 8192) = 79
pipe([3, 4])              = 0
clone(child_stack=NULL,
flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD,
child_tidptr=0x7f0d37894850) = 19164
close(4)                = 0
read(3, "/usr/sbin/lvcreate\n", 128)  = 19
--- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=19164, si_uid=0,
si_status=0, si_utime=0, si_stime=0} ---
rt_sigreturn({mask=[]})         = 19
read(3, "", 128)            = 0
close(3)                = 0
wait4(-1, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], 0, NULL) = 19164
stat("/usr/local/sbin/chroot", 0x7ffdd8247710) = -1 ENOENT (No such file or
directory)
stat("/usr/local/bin/chroot", 0x7ffdd8247710) = -1 ENOENT (No such file or
directory)
stat("/usr/sbin/chroot", {st_mode=S_IFREG|0755, st_size=43352, ...}) = 0
clone(child_stack=NULL,
flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD,
child_tidptr=0x7f0d37894850) = 19168
wait4(-1, ^Cstrace: Process 19163 detached
 <detached ...>

# ^C

`lvs` output on the node after pressing ctrl+c on the pod
container's terminal:
ubuntu@ip-172-31-85-197:~$ sudo lvs
 LV                    VG       Attr    LSize  Pool      Origin Data% Meta%
Move Log Cpy%Sync Convert
 test-lv                 shared-vg    -wi-a-----  1.00g

If ctrl+c is not pressed and trying to see the above output on the node
also hangs until the pod operation is interrupted.

[-- Attachment #1.2: Type: text/html, Size: 6457 bytes --]

[-- Attachment #2: Type: text/plain, Size: 202 bytes --]

_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/

Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[Roger Heflin]

[-- Attachment #1.1: Type: text/plain, Size: 6277 bytes --]

You need to rerun with an "strace -f".  That way it will also strace the
fork's one of which it appears to be waiting on.

On Mon, May 30, 2022 at 1:52 AM Abhishek Agarwal <
mragarwal.developer@gmail.com> wrote:

> When a kubernetes pod is scheduled on the node having lvm2 libraries
> already installed and trying to run lvm commands using those node binaries
> from inside the pod container, the commands hang and are waiting on
> something to complete. Although when ctrl+c is pressed the terminal session
> resumes and checking the final code for the execution returns a "0" error
> code and the commands operation is also carried out successfully.
>
>
> Below is the command output and strace for the command:-
>
> strace of lvcreate on the pod container scheduled on the node where lvm2
> binaries are present:
> # strace /sbin/lvm-eg/lvcreate -n test-lv -L 1G shared-vg
> execve("/sbin/lvm-eg/lvcreate", ["/sbin/lvm-eg/lvcreate", "-n", "test-lv",
> "-L", "1G", "shared-vg"], 0x7ffe7d7a6c98 /* 37 vars */) = 0
> brk(NULL)                = 0x55d196f41000
> arch_prctl(0x3001 /* ARCH_??? */, 0x7ffdd8247a80) = -1 EINVAL (Invalid
> argument)
> access("/etc/ld.so.preload", R_OK)   = -1 ENOENT (No such file or
> directory)
> openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
> fstat(3, {st_mode=S_IFREG|0644, st_size=8679, ...}) = 0
> mmap(NULL, 8679, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f0d37895000
> close(3)                = 0
> openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
> read(3,
> "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360A\2\0\0\0\0\0"..., 832)
> = 832
> pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"...,
> 784, 64) = 784
> pread64(3,
> "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32,
> 848) = 32
> pread64(3,
> "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\237\333t\347\262\27\320l\223\27*\202C\370T\177"...,
> 68, 880) = 68
> fstat(3, {st_mode=S_IFREG|0755, st_size=2029560, ...}) = 0
> mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
> 0x7f0d37893000
> pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"...,
> 784, 64) = 784
> pread64(3,
> "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32,
> 848) = 32
> pread64(3,
> "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\237\333t\347\262\27\320l\223\27*\202C\370T\177"...,
> 68, 880) = 68
> mmap(NULL, 2037344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
> 0x7f0d376a1000
> mmap(0x7f0d376c3000, 1540096, PROT_READ|PROT_EXEC,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7f0d376c3000
> mmap(0x7f0d3783b000, 319488, PROT_READ,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19a000) = 0x7f0d3783b000
> mmap(0x7f0d37889000, 24576, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7f0d37889000
> mmap(0x7f0d3788f000, 13920, PROT_READ|PROT_WRITE,
> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0d3788f000
> close(3)                = 0
> arch_prctl(ARCH_SET_FS, 0x7f0d37894580) = 0
> mprotect(0x7f0d37889000, 16384, PROT_READ) = 0
> mprotect(0x55d196662000, 8192, PROT_READ) = 0
> mprotect(0x7f0d378c5000, 4096, PROT_READ) = 0
> munmap(0x7f0d37895000, 8679)      = 0
> getuid()                = 0
> getgid()                = 0
> getpid()                = 19163
> rt_sigaction(SIGCHLD, {sa_handler=0x55d196657c30, sa_mask=~[RTMIN RT_1],
> sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
> geteuid()                = 0
> brk(NULL)                = 0x55d196f41000
> brk(0x55d196f62000)           = 0x55d196f62000
> getppid()                = 19160
> stat("/", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
> stat(".", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
> openat(AT_FDCWD, "/sbin/lvm-eg/lvcreate", O_RDONLY) = 3
> fcntl(3, F_DUPFD, 10)          = 10
> close(3)                = 0
> fcntl(10, F_SETFD, FD_CLOEXEC)     = 0
> geteuid()                = 0
> getegid()                = 0
> rt_sigaction(SIGINT, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0},
> 8) = 0
> rt_sigaction(SIGINT, {sa_handler=0x55d196657c30, sa_mask=~[RTMIN RT_1],
> sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
> rt_sigaction(SIGQUIT, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0},
> 8) = 0
> rt_sigaction(SIGQUIT, {sa_handler=SIG_DFL, sa_mask=~[RTMIN RT_1],
> sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
> rt_sigaction(SIGTERM, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0},
> 8) = 0
> rt_sigaction(SIGTERM, {sa_handler=SIG_DFL, sa_mask=~[RTMIN RT_1],
> sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
> read(10, "#!/bin/sh\npath=$(/sbin/lvm-eg/ge"..., 8192) = 79
> pipe([3, 4])              = 0
> clone(child_stack=NULL,
> flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD,
> child_tidptr=0x7f0d37894850) = 19164
> close(4)                = 0
> read(3, "/usr/sbin/lvcreate\n", 128)  = 19
> --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=19164, si_uid=0,
> si_status=0, si_utime=0, si_stime=0} ---
> rt_sigreturn({mask=[]})         = 19
> read(3, "", 128)            = 0
> close(3)                = 0
> wait4(-1, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], 0, NULL) = 19164
> stat("/usr/local/sbin/chroot", 0x7ffdd8247710) = -1 ENOENT (No such file
> or directory)
> stat("/usr/local/bin/chroot", 0x7ffdd8247710) = -1 ENOENT (No such file or
> directory)
> stat("/usr/sbin/chroot", {st_mode=S_IFREG|0755, st_size=43352, ...}) = 0
> clone(child_stack=NULL,
> flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD,
> child_tidptr=0x7f0d37894850) = 19168
> wait4(-1, ^Cstrace: Process 19163 detached
>  <detached ...>
>
> # ^C
>
> `lvs` output on the node after pressing ctrl+c on the pod
> container's terminal:
> ubuntu@ip-172-31-85-197:~$ sudo lvs
>  LV                    VG       Attr    LSize  Pool      Origin Data%
> Meta% Move Log Cpy%Sync Convert
>  test-lv                 shared-vg    -wi-a-----  1.00g
>
> If ctrl+c is not pressed and trying to see the above output on the node
> also hangs until the pod operation is interrupted.
> _______________________________________________
> linux-lvm mailing list
> linux-lvm@redhat.com
> https://listman.redhat.com/mailman/listinfo/linux-lvm
> read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/
>

[-- Attachment #1.2: Type: text/html, Size: 7443 bytes --]

[-- Attachment #2: Type: text/plain, Size: 202 bytes --]

_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/

Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[Zdenek Kabelac]

Dne 27. 05. 22 v 9:02 Abhishek Agarwal napsal(a):
> When a kubernetes pod is scheduled on the node having lvm2 libraries already 
> installed and trying to run lvm commands using those node binaries from inside 
> the pod container, the commands hang and are waiting on something to complete. 
> Although when ctrl+c is pressed the terminal session resumes and checking the 
> final code for the execution returns a "0" error code and the commands 
> operation is also carried out successfully.
> 
> 

lvm2 is *NOT* designed to be executed in/from a container.

It cannot work properly as it directly communicates with system's udevd - 
which you likely don't have running in your container.

You could kind of 'fake it' by running lvm2 wihout udev synchronization - but 
this will just open another cave of other problems (missing synchronization).

So your lvm2 command should be always executed on your hosting machine
(since it does control resources without containerization support - like 
devices) and then you should 'pass' created LV to  your container in some way.

Regards

Zdenek

_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/

Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[Zdenek Kabelac]

Dne 27. 05. 22 v 9:02 Abhishek Agarwal napsal(a):
> When a kubernetes pod is scheduled on the node having lvm2 libraries already 
> installed and trying to run lvm commands using those node binaries from inside 
> the pod container, the commands hang and are waiting on something to complete. 
> Although when ctrl+c is pressed the terminal session resumes and checking the 
> final code for the execution returns a "0" error code and the commands 
> operation is also carried out successfully.
> 
> 

lvm2 is *NOT* designed to be executed in/from a container.

It cannot work properly as it directly communicates with system's udevd - 
which you likely don't have running in your container.

You could kind of 'fake it' by running lvm2 wihout udev synchronization - but 
this will just open another cave of other problems (missing synchronization).

So your lvm2 command should be always executed on your hosting machine
(since it does control resources without containerization support - like 
devices) and then you should 'pass' created LV to  your container in some way.

Regards

Zdenek

_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/

Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[Abhishek Agarwal]

[-- Attachment #1.1: Type: text/plain, Size: 6931 bytes --]

Hi Roger. Thanks for your reply. I have rerun the command with `strace -f`
as you suggested. Here is the pastebin link containing the detailed output
of the command: https://pastebin.com/raw/VRuBbHBc

Thanks & Regards

On Tue, 31 May 2022 at 11:22, Roger Heflin <rogerheflin@gmail.com> wrote:

> You need to rerun with an "strace -f".  That way it will also strace the
> fork's one of which it appears to be waiting on.
>
> On Mon, May 30, 2022 at 1:52 AM Abhishek Agarwal <
> mragarwal.developer@gmail.com> wrote:
>
>> When a kubernetes pod is scheduled on the node having lvm2 libraries
>> already installed and trying to run lvm commands using those node binaries
>> from inside the pod container, the commands hang and are waiting on
>> something to complete. Although when ctrl+c is pressed the terminal session
>> resumes and checking the final code for the execution returns a "0" error
>> code and the commands operation is also carried out successfully.
>>
>>
>> Below is the command output and strace for the command:-
>>
>> strace of lvcreate on the pod container scheduled on the node where lvm2
>> binaries are present:
>> # strace /sbin/lvm-eg/lvcreate -n test-lv -L 1G shared-vg
>> execve("/sbin/lvm-eg/lvcreate", ["/sbin/lvm-eg/lvcreate", "-n",
>> "test-lv", "-L", "1G", "shared-vg"], 0x7ffe7d7a6c98 /* 37 vars */) = 0
>> brk(NULL)                = 0x55d196f41000
>> arch_prctl(0x3001 /* ARCH_??? */, 0x7ffdd8247a80) = -1 EINVAL (Invalid
>> argument)
>> access("/etc/ld.so.preload", R_OK)   = -1 ENOENT (No such file or
>> directory)
>> openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
>> fstat(3, {st_mode=S_IFREG|0644, st_size=8679, ...}) = 0
>> mmap(NULL, 8679, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f0d37895000
>> close(3)                = 0
>> openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) =
>> 3
>> read(3,
>> "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360A\2\0\0\0\0\0"..., 832)
>> = 832
>> pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"...,
>> 784, 64) = 784
>> pread64(3,
>> "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32,
>> 848) = 32
>> pread64(3,
>> "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\237\333t\347\262\27\320l\223\27*\202C\370T\177"...,
>> 68, 880) = 68
>> fstat(3, {st_mode=S_IFREG|0755, st_size=2029560, ...}) = 0
>> mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0)
>> = 0x7f0d37893000
>> pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"...,
>> 784, 64) = 784
>> pread64(3,
>> "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32,
>> 848) = 32
>> pread64(3,
>> "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\237\333t\347\262\27\320l\223\27*\202C\370T\177"...,
>> 68, 880) = 68
>> mmap(NULL, 2037344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
>> 0x7f0d376a1000
>> mmap(0x7f0d376c3000, 1540096, PROT_READ|PROT_EXEC,
>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7f0d376c3000
>> mmap(0x7f0d3783b000, 319488, PROT_READ,
>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19a000) = 0x7f0d3783b000
>> mmap(0x7f0d37889000, 24576, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7f0d37889000
>> mmap(0x7f0d3788f000, 13920, PROT_READ|PROT_WRITE,
>> MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0d3788f000
>> close(3)                = 0
>> arch_prctl(ARCH_SET_FS, 0x7f0d37894580) = 0
>> mprotect(0x7f0d37889000, 16384, PROT_READ) = 0
>> mprotect(0x55d196662000, 8192, PROT_READ) = 0
>> mprotect(0x7f0d378c5000, 4096, PROT_READ) = 0
>> munmap(0x7f0d37895000, 8679)      = 0
>> getuid()                = 0
>> getgid()                = 0
>> getpid()                = 19163
>> rt_sigaction(SIGCHLD, {sa_handler=0x55d196657c30, sa_mask=~[RTMIN RT_1],
>> sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
>> geteuid()                = 0
>> brk(NULL)                = 0x55d196f41000
>> brk(0x55d196f62000)           = 0x55d196f62000
>> getppid()                = 19160
>> stat("/", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
>> stat(".", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
>> openat(AT_FDCWD, "/sbin/lvm-eg/lvcreate", O_RDONLY) = 3
>> fcntl(3, F_DUPFD, 10)          = 10
>> close(3)                = 0
>> fcntl(10, F_SETFD, FD_CLOEXEC)     = 0
>> geteuid()                = 0
>> getegid()                = 0
>> rt_sigaction(SIGINT, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0},
>> 8) = 0
>> rt_sigaction(SIGINT, {sa_handler=0x55d196657c30, sa_mask=~[RTMIN RT_1],
>> sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
>> rt_sigaction(SIGQUIT, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0},
>> 8) = 0
>> rt_sigaction(SIGQUIT, {sa_handler=SIG_DFL, sa_mask=~[RTMIN RT_1],
>> sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
>> rt_sigaction(SIGTERM, NULL, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0},
>> 8) = 0
>> rt_sigaction(SIGTERM, {sa_handler=SIG_DFL, sa_mask=~[RTMIN RT_1],
>> sa_flags=SA_RESTORER, sa_restorer=0x7f0d376e40c0}, NULL, 8) = 0
>> read(10, "#!/bin/sh\npath=$(/sbin/lvm-eg/ge"..., 8192) = 79
>> pipe([3, 4])              = 0
>> clone(child_stack=NULL,
>> flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD,
>> child_tidptr=0x7f0d37894850) = 19164
>> close(4)                = 0
>> read(3, "/usr/sbin/lvcreate\n", 128)  = 19
>> --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=19164,
>> si_uid=0, si_status=0, si_utime=0, si_stime=0} ---
>> rt_sigreturn({mask=[]})         = 19
>> read(3, "", 128)            = 0
>> close(3)                = 0
>> wait4(-1, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], 0, NULL) = 19164
>> stat("/usr/local/sbin/chroot", 0x7ffdd8247710) = -1 ENOENT (No such file
>> or directory)
>> stat("/usr/local/bin/chroot", 0x7ffdd8247710) = -1 ENOENT (No such file
>> or directory)
>> stat("/usr/sbin/chroot", {st_mode=S_IFREG|0755, st_size=43352, ...}) = 0
>> clone(child_stack=NULL,
>> flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD,
>> child_tidptr=0x7f0d37894850) = 19168
>> wait4(-1, ^Cstrace: Process 19163 detached
>>  <detached ...>
>>
>> # ^C
>>
>> `lvs` output on the node after pressing ctrl+c on the pod
>> container's terminal:
>> ubuntu@ip-172-31-85-197:~$ sudo lvs
>>  LV                    VG       Attr    LSize  Pool      Origin Data%
>> Meta% Move Log Cpy%Sync Convert
>>  test-lv                 shared-vg    -wi-a-----  1.00g
>>
>> If ctrl+c is not pressed and trying to see the above output on the node
>> also hangs until the pod operation is interrupted.
>> _______________________________________________
>> linux-lvm mailing list
>> linux-lvm@redhat.com
>> https://listman.redhat.com/mailman/listinfo/linux-lvm
>> read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/
>>
> _______________________________________________
> linux-lvm mailing list
> linux-lvm@redhat.com
> https://listman.redhat.com/mailman/listinfo/linux-lvm
> read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/
>

[-- Attachment #1.2: Type: text/html, Size: 8662 bytes --]

[-- Attachment #2: Type: text/plain, Size: 202 bytes --]

_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/

Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[Demi Marie Obenour]

[-- Attachment #1.1: Type: text/plain, Size: 645 bytes --]

On Wed, Jun 01, 2022 at 12:20:32AM +0530, Abhishek Agarwal wrote:
> Hi Roger. Thanks for your reply. I have rerun the command with `strace -f`
> as you suggested. Here is the pastebin link containing the detailed output
> of the command: https://pastebin.com/raw/VRuBbHBc

Even if you can get LVM “working”, it is still likely to cause data
corruption at some point, as there is no guarantee that different LVM
processes in different namespaces will see each others’ locks.

Why do you need to run LVM in a container?  What are you trying to
accomplish?
-- 
Sincerely,
Demi Marie Obenour (she/her/hers)
Invisible Things Lab

[-- Attachment #1.2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

[-- Attachment #2: Type: text/plain, Size: 202 bytes --]

_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/

Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[Abhishek Agarwal]

[-- Attachment #1.1: Type: text/plain, Size: 2433 bytes --]

These are not different LVM processes. The container process is using the
LVM binary that the node itself has. We have achieved this by using scripts
that point to the same lvm binary that is used by the node.

Configmap(~shell script) used for the same has the following contents where
`/host` refers to the root directory of the node:

get_bin_path: |
  #!/bin/sh
  bin_name=$1
  if [ -x /host/bin/which ]; then
    echo $(chroot /host /bin/which $bin_name | cut -d ' ' -f 1)
  elif [ -x /host/usr/bin/which ]; then
    echo $(chroot /host /usr/bin/which $bin_name | cut -d ' ' -f 1)
  else
    $(chroot /host which $bin_name | cut -d ' ' -f 1)
  fi

lvcreate: |
  #!/bin/sh
  path=$(/sbin/lvm-eg/get_bin_path "lvcreate")
  chroot /host $path "$@"

Also, the above logs in the pastebin link have errors because the vg lock
has not been acquired and hence creation commands will fail. Once the lock
is acquired, the `strace -f` command gives the following output being
stuck. Check out this link for full details ->
https://pastebin.com/raw/DwQfdmr8

P.S: We at OpenEBS are trying to provide lvm storage to cloud native
workloads with the help of kubernetes CSI drivers and since all these
drivers run as pods and help dynamic provisioning of kubernetes
volumes(storage) for the application, the lvm commands needs to be run from
inside the pod. Reference -> https://github.com/openebs/lvm-localpv

Regards

On Wed, 1 Jun 2022 at 13:06, Demi Marie Obenour <demi@invisiblethingslab.com>
wrote:

> On Wed, Jun 01, 2022 at 12:20:32AM +0530, Abhishek Agarwal wrote:
> > Hi Roger. Thanks for your reply. I have rerun the command with `strace
> -f`
> > as you suggested. Here is the pastebin link containing the detailed
> output
> > of the command: https://pastebin.com/raw/VRuBbHBc
>
> Even if you can get LVM “working”, it is still likely to cause data
> corruption at some point, as there is no guarantee that different LVM
> processes in different namespaces will see each others’ locks.
>
> Why do you need to run LVM in a container?  What are you trying to
> accomplish?
> --
> Sincerely,
> Demi Marie Obenour (she/her/hers)
> Invisible Things Lab
> _______________________________________________
> linux-lvm mailing list
> linux-lvm@redhat.com
> https://listman.redhat.com/mailman/listinfo/linux-lvm
> read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/
>

[-- Attachment #1.2: Type: text/html, Size: 3737 bytes --]

[-- Attachment #2: Type: text/plain, Size: 202 bytes --]

_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/

Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[Roger Heflin]

[-- Attachment #1.1: Type: text/plain, Size: 3113 bytes --]

Random thoughts.

Make sure  use_lvmetad is 0, and its systemd units for it are
stopped/disabled.

Are you mounting /proc and /sys and /dev into the /host chroot?

/run may also be needed.

you might add a "-ttt" to the strace command to give timing data.



On Thu, Jun 2, 2022 at 1:41 AM Abhishek Agarwal <
mragarwal.developer@gmail.com> wrote:

> These are not different LVM processes. The container process is using the
> LVM binary that the node itself has. We have achieved this by using scripts
> that point to the same lvm binary that is used by the node.
>
> Configmap(~shell script) used for the same has the following contents
> where `/host` refers to the root directory of the node:
>
> get_bin_path: |
>   #!/bin/sh
>   bin_name=$1
>   if [ -x /host/bin/which ]; then
>     echo $(chroot /host /bin/which $bin_name | cut -d ' ' -f 1)
>   elif [ -x /host/usr/bin/which ]; then
>     echo $(chroot /host /usr/bin/which $bin_name | cut -d ' ' -f 1)
>   else
>     $(chroot /host which $bin_name | cut -d ' ' -f 1)
>   fi
>
> lvcreate: |
>   #!/bin/sh
>   path=$(/sbin/lvm-eg/get_bin_path "lvcreate")
>   chroot /host $path "$@"
>
> Also, the above logs in the pastebin link have errors because the vg lock
> has not been acquired and hence creation commands will fail. Once the lock
> is acquired, the `strace -f` command gives the following output being
> stuck. Check out this link for full details ->
> https://pastebin.com/raw/DwQfdmr8
>
> P.S: We at OpenEBS are trying to provide lvm storage to cloud native
> workloads with the help of kubernetes CSI drivers and since all these
> drivers run as pods and help dynamic provisioning of kubernetes
> volumes(storage) for the application, the lvm commands needs to be run from
> inside the pod. Reference -> https://github.com/openebs/lvm-localpv
>
> Regards
>
> On Wed, 1 Jun 2022 at 13:06, Demi Marie Obenour <
> demi@invisiblethingslab.com> wrote:
>
>> On Wed, Jun 01, 2022 at 12:20:32AM +0530, Abhishek Agarwal wrote:
>> > Hi Roger. Thanks for your reply. I have rerun the command with `strace
>> -f`
>> > as you suggested. Here is the pastebin link containing the detailed
>> output
>> > of the command: https://pastebin.com/raw/VRuBbHBc
>>
>> Even if you can get LVM “working”, it is still likely to cause data
>> corruption at some point, as there is no guarantee that different LVM
>> processes in different namespaces will see each others’ locks.
>>
>> Why do you need to run LVM in a container?  What are you trying to
>> accomplish?
>> --
>> Sincerely,
>> Demi Marie Obenour (she/her/hers)
>> Invisible Things Lab
>> _______________________________________________
>> linux-lvm mailing list
>> linux-lvm@redhat.com
>> https://listman.redhat.com/mailman/listinfo/linux-lvm
>> read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/
>>
> _______________________________________________
> linux-lvm mailing list
> linux-lvm@redhat.com
> https://listman.redhat.com/mailman/listinfo/linux-lvm
> read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/
>

[-- Attachment #1.2: Type: text/html, Size: 4991 bytes --]

[-- Attachment #2: Type: text/plain, Size: 202 bytes --]

_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/

Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[Nir Soffer]

On Thu, Jun 2, 2022 at 9:41 AM Abhishek Agarwal
<mragarwal.developer@gmail.com> wrote:
>
> These are not different LVM processes. The container process is using the LVM binary that the node itself has. We have achieved this by using scripts that point to the same lvm binary that is used by the node.
>
> Configmap(~shell script) used for the same has the following contents where `/host` refers to the root directory of the node:
>
> get_bin_path: |
>   #!/bin/sh
>   bin_name=$1
>   if [ -x /host/bin/which ]; then
>     echo $(chroot /host /bin/which $bin_name | cut -d ' ' -f 1)
>   elif [ -x /host/usr/bin/which ]; then
>     echo $(chroot /host /usr/bin/which $bin_name | cut -d ' ' -f 1)
>   else
>     $(chroot /host which $bin_name | cut -d ' ' -f 1)
>   fi
>
> lvcreate: |
>   #!/bin/sh
>   path=$(/sbin/lvm-eg/get_bin_path "lvcreate")
>   chroot /host $path "$@"
>
> Also, the above logs in the pastebin link have errors because the vg lock has not been acquired and hence creation commands will fail. Once the lock is acquired, the `strace -f` command gives the following output being stuck. Check out this link for full details -> https://pastebin.com/raw/DwQfdmr8
>
> P.S: We at OpenEBS are trying to provide lvm storage to cloud native workloads with the help of kubernetes CSI drivers and since all these drivers run as pods and help dynamic provisioning of kubernetes volumes(storage) for the application, the lvm commands needs to be run from inside the pod. Reference -> https://github.com/openebs/lvm-localpv

For using LVM in Kubernetes, why not toplvm?
https://github.com/topolvm/topolvm

The design looks right, running lvm commands on the host:
https://github.com/topolvm/topolvm/blob/main/docs/design.md

Nir

_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/

Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[Abhishek Agarwal]

[-- Attachment #1.1: Type: text/plain, Size: 4183 bytes --]

1. Yes, use_lvmetad is 0, and its systemd units for it are stopped/disabled.
2. Yes, everything on the host machine i.e(/proc, /sys etc) are getting
mounted on the pod.

*ubuntu@ip-172-31-89-47*:*~*$ kubectl exec -it openebs-lvm-node-v6jrb -c
openebs-lvm-plugin  -n kube-system -- sh

# ls

bin  boot  dev etc  home  host  lib  lib32  lib64  libx32  media  mnt opt
plugin  proc  root  run  sbin  srv  sys  tmp  usr var

# cd /host

# ls

bin  boot  dev etc  home  lib lib32  lib64  libx32  lost+found  media  mnt
opt  proc  root  run  sbin  snap srv  sys  tmp  usr  var

#
3. The detail output of `strace -f -ttt` command:
https://pastebin.com/raw/VFyXLNaC

On Fri, 3 Jun 2022 at 12:48, Roger Heflin <rogerheflin@gmail.com> wrote:

> Random thoughts.
>
> Make sure  use_lvmetad is 0, and its systemd units for it are
> stopped/disabled.
>
> Are you mounting /proc and /sys and /dev into the /host chroot?
>
> /run may also be needed.
>
> you might add a "-ttt" to the strace command to give timing data.
>
>
>
> On Thu, Jun 2, 2022 at 1:41 AM Abhishek Agarwal <
> mragarwal.developer@gmail.com> wrote:
>
>> These are not different LVM processes. The container process is using the
>> LVM binary that the node itself has. We have achieved this by using scripts
>> that point to the same lvm binary that is used by the node.
>>
>> Configmap(~shell script) used for the same has the following contents
>> where `/host` refers to the root directory of the node:
>>
>> get_bin_path: |
>>   #!/bin/sh
>>   bin_name=$1
>>   if [ -x /host/bin/which ]; then
>>     echo $(chroot /host /bin/which $bin_name | cut -d ' ' -f 1)
>>   elif [ -x /host/usr/bin/which ]; then
>>     echo $(chroot /host /usr/bin/which $bin_name | cut -d ' ' -f 1)
>>   else
>>     $(chroot /host which $bin_name | cut -d ' ' -f 1)
>>   fi
>>
>> lvcreate: |
>>   #!/bin/sh
>>   path=$(/sbin/lvm-eg/get_bin_path "lvcreate")
>>   chroot /host $path "$@"
>>
>> Also, the above logs in the pastebin link have errors because the vg lock
>> has not been acquired and hence creation commands will fail. Once the lock
>> is acquired, the `strace -f` command gives the following output being
>> stuck. Check out this link for full details ->
>> https://pastebin.com/raw/DwQfdmr8
>>
>> P.S: We at OpenEBS are trying to provide lvm storage to cloud native
>> workloads with the help of kubernetes CSI drivers and since all these
>> drivers run as pods and help dynamic provisioning of kubernetes
>> volumes(storage) for the application, the lvm commands needs to be run from
>> inside the pod. Reference -> https://github.com/openebs/lvm-localpv
>>
>> Regards
>>
>> On Wed, 1 Jun 2022 at 13:06, Demi Marie Obenour <
>> demi@invisiblethingslab.com> wrote:
>>
>>> On Wed, Jun 01, 2022 at 12:20:32AM +0530, Abhishek Agarwal wrote:
>>> > Hi Roger. Thanks for your reply. I have rerun the command with `strace
>>> -f`
>>> > as you suggested. Here is the pastebin link containing the detailed
>>> output
>>> > of the command: https://pastebin.com/raw/VRuBbHBc
>>>
>>> Even if you can get LVM “working”, it is still likely to cause data
>>> corruption at some point, as there is no guarantee that different LVM
>>> processes in different namespaces will see each others’ locks.
>>>
>>> Why do you need to run LVM in a container?  What are you trying to
>>> accomplish?
>>> --
>>> Sincerely,
>>> Demi Marie Obenour (she/her/hers)
>>> Invisible Things Lab
>>> _______________________________________________
>>> linux-lvm mailing list
>>> linux-lvm@redhat.com
>>> https://listman.redhat.com/mailman/listinfo/linux-lvm
>>> read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/
>>>
>> _______________________________________________
>> linux-lvm mailing list
>> linux-lvm@redhat.com
>> https://listman.redhat.com/mailman/listinfo/linux-lvm
>> read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/
>>
> _______________________________________________
> linux-lvm mailing list
> linux-lvm@redhat.com
> https://listman.redhat.com/mailman/listinfo/linux-lvm
> read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/
>

[-- Attachment #1.2: Type: text/html, Size: 10830 bytes --]

[-- Attachment #2: Type: text/plain, Size: 202 bytes --]

_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/

Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[Demi Marie Obenour]

[-- Attachment #1.1: Type: text/plain, Size: 1053 bytes --]

On Mon, Jun 06, 2022 at 11:19:47AM +0530, Abhishek Agarwal wrote:
> 1. Yes, use_lvmetad is 0, and its systemd units for it are stopped/disabled.
> 2. Yes, everything on the host machine i.e(/proc, /sys etc) are getting
> mounted on the pod.
> 
> *ubuntu@ip-172-31-89-47*:*~*$ kubectl exec -it openebs-lvm-node-v6jrb -c
> openebs-lvm-plugin  -n kube-system -- sh
> 
> # ls
> 
> bin  boot  dev etc  home  host  lib  lib32  lib64  libx32  media  mnt opt
> plugin  proc  root  run  sbin  srv  sys  tmp  usr var
> 
> # cd /host
> 
> # ls
> 
> bin  boot  dev etc  home  lib lib32  lib64  libx32  lost+found  media  mnt
> opt  proc  root  run  sbin  snap srv  sys  tmp  usr  var
> 
> #
> 3. The detail output of `strace -f -ttt` command:
> https://pastebin.com/raw/VFyXLNaC

I suggest bind-mounting the host’s D-Bus socket into the container and
using systemd’s D-Bus API to run the LVM commands on the host.  This
will avoid the problems you are having.
-- 
Sincerely,
Demi Marie Obenour (she/her/hers)
Invisible Things Lab

[-- Attachment #1.2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

[-- Attachment #2: Type: text/plain, Size: 202 bytes --]

_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/

Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[Bernd Eckenfels]

[-- Attachment #1.1: Type: text/plain, Size: 4731 bytes --]

Not sure if relevant, but this could be a kernel (hidepid?) hardening or empty mount?


[pid   360] 1654493537.688190 getppid() = 355
[pid   360] 1654493537.688437 openat(AT_FDCWD, "/proc/355/cmdline", O_RDONLY) = -1 ENOENT (No such file or directory)

Gruss
Bernd


--
http://bernd.eckenfels.net
________________________________
Von: linux-lvm <linux-lvm-bounces@redhat.com> im Auftrag von Abhishek Agarwal <mragarwal.developer@gmail.com>
Gesendet: Monday, June 6, 2022 7:49:47 AM
An: LVM general discussion and development <linux-lvm@redhat.com>
Betreff: Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod

1. Yes, use_lvmetad is 0, and its systemd units for it are stopped/disabled.
2. Yes, everything on the host machine i.e(/proc, /sys etc) are getting mounted on the pod.

ubuntu@ip-172-31-89-47:~$ kubectl exec -it openebs-lvm-node-v6jrb -c openebs-lvm-plugin  -n kube-system -- sh

# ls

bin  boot  dev etc  home  host  lib  lib32  lib64  libx32  media  mnt opt  plugin  proc  root  run  sbin  srv  sys  tmp  usr var

# cd /host

# ls

bin  boot  dev etc  home  lib lib32  lib64  libx32  lost+found  media  mnt  opt  proc  root  run  sbin  snap srv  sys  tmp  usr  var

#

3. The detail output of `strace -f -ttt` command: https://pastebin.com/raw/VFyXLNaC

On Fri, 3 Jun 2022 at 12:48, Roger Heflin <rogerheflin@gmail.com<mailto:rogerheflin@gmail.com>> wrote:
Random thoughts.

Make sure  use_lvmetad is 0, and its systemd units for it are stopped/disabled.

Are you mounting /proc and /sys and /dev into the /host chroot?

/run may also be needed.

you might add a "-ttt" to the strace command to give timing data.



On Thu, Jun 2, 2022 at 1:41 AM Abhishek Agarwal <mragarwal.developer@gmail.com<mailto:mragarwal.developer@gmail.com>> wrote:
These are not different LVM processes. The container process is using the LVM binary that the node itself has. We have achieved this by using scripts that point to the same lvm binary that is used by the node.

Configmap(~shell script) used for the same has the following contents where `/host` refers to the root directory of the node:

get_bin_path: |
  #!/bin/sh
  bin_name=$1
  if [ -x /host/bin/which ]; then
    echo $(chroot /host /bin/which $bin_name | cut -d ' ' -f 1)
  elif [ -x /host/usr/bin/which ]; then
    echo $(chroot /host /usr/bin/which $bin_name | cut -d ' ' -f 1)
  else
    $(chroot /host which $bin_name | cut -d ' ' -f 1)
  fi

lvcreate: |
  #!/bin/sh
  path=$(/sbin/lvm-eg/get_bin_path "lvcreate")
  chroot /host $path "$@"

Also, the above logs in the pastebin link have errors because the vg lock has not been acquired and hence creation commands will fail. Once the lock is acquired, the `strace -f` command gives the following output being stuck. Check out this link for full details -> https://pastebin.com/raw/DwQfdmr8

P.S: We at OpenEBS are trying to provide lvm storage to cloud native workloads with the help of kubernetes CSI drivers and since all these drivers run as pods and help dynamic provisioning of kubernetes volumes(storage) for the application, the lvm commands needs to be run from inside the pod. Reference -> https://github.com/openebs/lvm-localpv

Regards

On Wed, 1 Jun 2022 at 13:06, Demi Marie Obenour <demi@invisiblethingslab.com<mailto:demi@invisiblethingslab.com>> wrote:
On Wed, Jun 01, 2022 at 12:20:32AM +0530, Abhishek Agarwal wrote:
> Hi Roger. Thanks for your reply. I have rerun the command with `strace -f`
> as you suggested. Here is the pastebin link containing the detailed output
> of the command: https://pastebin.com/raw/VRuBbHBc

Even if you can get LVM “working”, it is still likely to cause data
corruption at some point, as there is no guarantee that different LVM
processes in different namespaces will see each others’ locks.

Why do you need to run LVM in a container?  What are you trying to
accomplish?
--
Sincerely,
Demi Marie Obenour (she/her/hers)
Invisible Things Lab
_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com<mailto:linux-lvm@redhat.com>
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/
_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com<mailto:linux-lvm@redhat.com>
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/
_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com<mailto:linux-lvm@redhat.com>
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/

[-- Attachment #1.2: Type: text/html, Size: 12783 bytes --]

[-- Attachment #2: Type: text/plain, Size: 202 bytes --]

_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/

Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[Abhishek Agarwal]

[-- Attachment #1.1: Type: text/plain, Size: 1653 bytes --]

Hey Demi, can you explain how it will help to solve the problem? I'm
actually not aware of that much low-level stuff but would like to learn
about it. Also, can you provide a few references for it on how I can use it?

Thanks

On Mon, 6 Jun 2022 at 12:32, Demi Marie Obenour <demi@invisiblethingslab.com>
wrote:

> On Mon, Jun 06, 2022 at 11:19:47AM +0530, Abhishek Agarwal wrote:
> > 1. Yes, use_lvmetad is 0, and its systemd units for it are
> stopped/disabled.
> > 2. Yes, everything on the host machine i.e(/proc, /sys etc) are getting
> > mounted on the pod.
> >
> > *ubuntu@ip-172-31-89-47*:*~*$ kubectl exec -it openebs-lvm-node-v6jrb -c
> > openebs-lvm-plugin  -n kube-system -- sh
> >
> > # ls
> >
> > bin  boot  dev etc  home  host  lib  lib32  lib64  libx32  media  mnt opt
> > plugin  proc  root  run  sbin  srv  sys  tmp  usr var
> >
> > # cd /host
> >
> > # ls
> >
> > bin  boot  dev etc  home  lib lib32  lib64  libx32  lost+found  media
> mnt
> > opt  proc  root  run  sbin  snap srv  sys  tmp  usr  var
> >
> > #
> > 3. The detail output of `strace -f -ttt` command:
> > https://pastebin.com/raw/VFyXLNaC
>
> I suggest bind-mounting the host’s D-Bus socket into the container and
> using systemd’s D-Bus API to run the LVM commands on the host.  This
> will avoid the problems you are having.
> --
> Sincerely,
> Demi Marie Obenour (she/her/hers)
> Invisible Things Lab
> _______________________________________________
> linux-lvm mailing list
> linux-lvm@redhat.com
> https://listman.redhat.com/mailman/listinfo/linux-lvm
> read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/
>

[-- Attachment #1.2: Type: text/html, Size: 2675 bytes --]

[-- Attachment #2: Type: text/plain, Size: 202 bytes --]

_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/

Re: [linux-lvm] lvm commands hanging when run from inside a kubernetes pod

[Demi Marie Obenour]

[-- Attachment #1.1: Type: text/plain, Size: 3845 bytes --]

On Mon, Jun 06, 2022 at 05:01:15PM +0530, Abhishek Agarwal wrote:
> Hey Demi, can you explain how it will help to solve the problem? I'm
> actually not aware of that much low-level stuff but would like to learn
> about it.

By default, a systemd unit runs in the same namespaces as systemd
itself.  Therefore, it runs outside of any container, and has full
access to udev and the host filesystem.  This is what you want when
running lvm2 commands.

> Also, can you provide a few references for it on how I can use it?

The easiest method is the systemd-run command-line tool.  I believe
“systemd-run --working-directory=/ --pipe --quiet -- lvm "$@"” should
work, with "$@" replaced by the actual LVM command you want to run.  Be
sure to pass --reportformat=json to get machine-readable JSON output.
The default output depends on configuration in /etc/lvm/lvm.conf, so you
don’t want to rely on it.  Alternatively, you can pass no arguments to
lvm and get an interactive shell, but that is a bit more complex to use.

To use this method, you will need to bind-mount the host’s system-wide
D-Bus instance into your container.  You will likely need to disable all
forms of security confinement and user namespacing as well.  This means
your container will have full control over the system, but LVM requires
full control over the system in order to function, so that does not
impact security much.  Your container can expose an API that impose
whatever restrictions it desires.

Instead of systemd-run, you can use the D-Bus API exposed by PID 1
directly, but that requires slightly more work than just calling a
command-line tool.  I have never used D-Bus from Go so I cannot comment
on how easy this is.

There are some other caveats with LVM.  I am not sure if these matter
for your use-case, but I thought you might want to be aware of them:

- LVM commands are slow (0.2 to 0.4 seconds or so) and serialized with a
  per-volume group lock.  Performance of individual commands is not a
  high priority of LVM upstream as per prior mailing list discussion.
  The actual time that I/O is suspended is much shorter.

- If LVM gets SIGKILLd or OOM-killed, your system may be left in an
  inconsistent state that requires a reboot to fix.  The latter can be
  prevented by setting OOMScoreAdjust to -1000.

- If you use thin provisioning (via thin pools and/or VDO), be sure to
  have monitoring so you can prevent out of space conditions.  Out of
  space conditions will likely result in all volumes going offline, and
  recovery may require growing the pool.

- Thin pools are backed by the dm-thin device-mapper target, which is
  optimized for overwriting already allocated blocks.  Writing to shared
  blocks, and possibly allocating new blocks, appears to triggers a slow
  path in dm-thin.  Discards are only supported at the block size
  granularity, which is typically greater than the block size of a
  filesystem.

- Deleting a thin volume does not pass down discards to the underlying
  block device, even if LVM is configured to discard deleted logical
  volumes.  You need to use blkdiscard before deleting the volume, but
  this can hang the entire pool unless you use the --step option to
  limit the amount of data discarded at once.

- If you are going to be exposing individual thinly-provisioned block
  devices to untrusted code (such as virtual machine guests), you need
  to prevent udev from scanning the thin volumes and keep zeroing of
  newly provisioned blocks enabled.  The latter is synchronous and slow.

- Shrinking thin or VDO pools is not supported.

- Old-style (not thin) snapshots are slow, and only intended for
  short-lived snapshots for backup purposes.
-- 
Sincerely,
Demi Marie Obenour (she/her/hers)
Invisible Things Lab

[-- Attachment #1.2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

[-- Attachment #2: Type: text/plain, Size: 202 bytes --]

_______________________________________________
linux-lvm mailing list
linux-lvm@redhat.com
https://listman.redhat.com/mailman/listinfo/linux-lvm
read the LVM HOW-TO at http://tldp.org/HOWTO/LVM-HOWTO/