BTRFS with RAID1 cannot boot when removing drive

BTRFS with RAID1 cannot boot when removing drive

[Saint Germain]

Hello,

I am experimenting with BTRFS and RAID1 on my Debian Wheezy (with
backported kernel 3.12-0.bpo.1-amd64) using a a motherboard with UEFI.

However I haven't managed to make the system boot when the removing the
first hard drive.

I have installed Debian with the following partition on the first hard
drive (no BTRFS subsystem):
/dev/sda1: for / (BTRFS)
/dev/sda2: for /home (BTRFS)
/dev/sda3: for swap

Then I added another drive for a RAID1 configuration (with btrfs
balance) and I installed grub on the second hard drive with
"grub-install /dev/sdb".

If I boot on sdb, it takes sda1 as the root filesystem
If I switched the cable, it always take the first hard drive as the
root filesystem (now sdb)
If I disconnect /dev/sda, the system doesn't boot with a message
saying that it hasn't found the UUID:

Scanning for BTRFS filesystems...
mount: mounting /dev/disk/by-uuid/c64fca2a-5700-4cca-abac-3a61f2f7486c on /root failed: Invalid argument

Can you tell me what I have done incorrectly ?
Is it because of UEFI ? If yes I haven't understood how I can correct
it in a simple way.

As extra question, I don't see also how I can configure the system to
get the correct swap in case of disk failure. Should I force both swap partition
to have the same UUID ?

Many thanks in advance !

****************************************************
Here are some outputs for info:

btrfs filesystem show
Label: none  uuid: 743d6b3b-71a7-4869-a0af-83549555284b
	Total devices 2 FS bytes used 27.96MB
	devid    1 size 897.98GB used 3.03GB path /dev/sda2
	devid    2 size 897.98GB used 3.03GB path /dev/sdb2

Label: none  uuid: c64fca2a-5700-4cca-abac-3a61f2f7486c
	Total devices 2 FS bytes used 3.85GB
	devid    1 size 27.94GB used 7.03GB path /dev/sda1
	devid    2 size 27.94GB used 7.03GB path /dev/sdb1

blkid 
/dev/sda1: UUID="c64fca2a-5700-4cca-abac-3a61f2f7486c" UUID_SUB="77ffad34-681c-4c43-9143-9b73da7d1ae3" TYPE="btrfs" 
/dev/sda3: UUID="469715b2-2fa3-4462-b6f5-62c04a60a4a2" TYPE="swap" 
/dev/sda2: UUID="743d6b3b-71a7-4869-a0af-83549555284b" UUID_SUB="744510f5-5bd5-4df4-b8c4-0fc1a853199a" TYPE="btrfs" 
/dev/sdb1: UUID="c64fca2a-5700-4cca-abac-3a61f2f7486c" UUID_SUB="2615fd98-f2ad-4e7b-84bc-0ee7f9770ca0" TYPE="btrfs" 
/dev/sdb2: UUID="743d6b3b-71a7-4869-a0af-83549555284b" UUID_SUB="8783a7b1-57ef-4bcc-ae7f-be20761e9a19" TYPE="btrfs" 
/dev/sdb3: UUID="56fbbe2f-7048-488f-b263-ab2eb000d1e1" TYPE="swap"

cat /etc/fstab
# <file system> <mount point>   <type>  <options>       <dump>  <pass>
UUID=c64fca2a-5700-4cca-abac-3a61f2f7486c /               btrfs   defaults        0       1
UUID=743d6b3b-71a7-4869-a0af-83549555284b /home           btrfs   defaults        0       2
UUID=469715b2-2fa3-4462-b6f5-62c04a60a4a2 none            swap    sw              0       0

cat /boot/grub/grub.cfg 
#
# DO NOT EDIT THIS FILE
#
# It is automatically generated by grub-mkconfig using templates
# from /etc/grub.d and settings from /etc/default/grub
#

### BEGIN /etc/grub.d/00_header ###
if [ -s $prefix/grubenv ]; then
  load_env
fi
set default="0"
if [ "${prev_saved_entry}" ]; then
  set saved_entry="${prev_saved_entry}"
  save_env saved_entry
  set prev_saved_entry=
  save_env prev_saved_entry
  set boot_once=true
fi

function savedefault {
  if [ -z "${boot_once}" ]; then
    saved_entry="${chosen}"
    save_env saved_entry
  fi
}

function load_video {
  insmod vbe
  insmod vga
  insmod video_bochs
  insmod video_cirrus
}

insmod part_msdos
insmod btrfs
set root='(hd1,msdos1)'
search --no-floppy --fs-uuid --set=root c64fca2a-5700-4cca-abac-3a61f2f7486c
if loadfont /usr/share/grub/unicode.pf2 ; then
  set gfxmode=640x480
  load_video
  insmod gfxterm
  insmod part_msdos
  insmod btrfs
  set root='(hd1,msdos1)'
  search --no-floppy --fs-uuid --set=root c64fca2a-5700-4cca-abac-3a61f2f7486c
  set locale_dir=($root)/boot/grub/locale
  set lang=fr_FR
  insmod gettext
fi
terminal_output gfxterm
set timeout=5
### END /etc/grub.d/00_header ###

### BEGIN /etc/grub.d/05_debian_theme ###
insmod part_msdos
insmod btrfs
set root='(hd1,msdos1)'
search --no-floppy --fs-uuid --set=root c64fca2a-5700-4cca-abac-3a61f2f7486c
insmod png
if background_image /usr/share/images/desktop-base/joy-grub.png; then
  set color_normal=white/black
  set color_highlight=black/white
else
  set menu_color_normal=cyan/blue
  set menu_color_highlight=white/blue
fi
### END /etc/grub.d/05_debian_theme ###

### BEGIN /etc/grub.d/10_linux ###
menuentry 'Debian GNU/Linux, with Linux 3.12-0.bpo.1-amd64' --class debian --class gnu-linux --class gnu --class os {
	load_video
	insmod gzio
	insmod part_msdos
	insmod btrfs
	set root='(hd1,msdos1)'
	search --no-floppy --fs-uuid --set=root c64fca2a-5700-4cca-abac-3a61f2f7486c
	echo	'Chargement de Linux 3.12-0.bpo.1-amd64 ...'
	linux	/boot/vmlinuz-3.12-0.bpo.1-amd64 root=UUID=c64fca2a-5700-4cca-abac-3a61f2f7486c ro  quiet
	echo	'Chargement du disque mémoire initial ...'
	initrd	/boot/initrd.img-3.12-0.bpo.1-amd64
}
menuentry 'Debian GNU/Linux, with Linux 3.12-0.bpo.1-amd64 (mode de dépannage)' --class debian --class gnu-linux --class gnu --class os {
	load_video
	insmod gzio
	insmod part_msdos
	insmod btrfs
	set root='(hd1,msdos1)'
	search --no-floppy --fs-uuid --set=root c64fca2a-5700-4cca-abac-3a61f2f7486c
	echo	'Chargement de Linux 3.12-0.bpo.1-amd64 ...'
	linux	/boot/vmlinuz-3.12-0.bpo.1-amd64 root=UUID=c64fca2a-5700-4cca-abac-3a61f2f7486c ro single 
	echo	'Chargement du disque mémoire initial ...'
	initrd	/boot/initrd.img-3.12-0.bpo.1-amd64
}
### END /etc/grub.d/10_linux ###

### BEGIN /etc/grub.d/20_linux_xen ###
### END /etc/grub.d/20_linux_xen ###

### BEGIN /etc/grub.d/30_os-prober ###
### END /etc/grub.d/30_os-prober ###

### BEGIN /etc/grub.d/40_custom ###
# This file provides an easy way to add custom menu entries.  Simply type the
# menu entries you want to add after this comment.  Be careful not to change
# the 'exec tail' line above.
### END /etc/grub.d/40_custom ###

### BEGIN /etc/grub.d/41_custom ###
if [ -f  $prefix/custom.cfg ]; then
  source $prefix/custom.cfg;
fi
### END /etc/grub.d/41_custom ###

Re: BTRFS with RAID1 cannot boot when removing drive

[Duncan]

Saint Germain posted on Sun, 09 Feb 2014 22:40:55 +0100 as excerpted:

> I am experimenting with BTRFS and RAID1 on my Debian Wheezy (with
> backported kernel 3.12-0.bpo.1-amd64) using a a motherboard with UEFI.

My systems don't do UEFI, but I do run GPT partitions and use grub2 for 
booting, with grub2-core installed to a BIOS/reserved type partition 
(instead of as an EFI service as it would be with UEFI).  And I have root 
filesystem btrfs two-device raid1 mode working fine here, tested bootable 
with only one device of the two available.

So while I can't help you directly with UEFI, I know the rest of it can/
does work.

One more thing:  I do have a (small) separate btrfs /boot, actually two 
of them as I setup a separate /boot on each of the two devices in ordered 
to have a backup /boot, since grub can only point to one /boot by 
default, and while pointing to another in grub's rescue mode is possible, 
I didn't want to have to deal with that if the first /boot was corrupted, 
as it's easier to simply point the BIOS at a different drive entirely and 
load its (independently installed and configured) grub and /boot.

But grub2's btrfs module reads raid1 mode just fine as I can access files 
on the btrfs raid1 mode rootfs directly from grub without issue, so 
that's not a problem.

But I strongly suspect I know what is... and it's a relatively easy fix.  
See below.  =:^)

> However I haven't managed to make the system boot when the removing the
> first hard drive.
> 
> I have installed Debian with the following partition on the first hard
> drive (no BTRFS subsystem):
> /dev/sda1: for / (BTRFS)
> /dev/sda2: for /home (BTRFS)
> /dev/sda3: for swap
> 
> Then I added another drive for a RAID1 configuration (with btrfs
> balance) and I installed grub on the second hard drive with
> "grub-install /dev/sdb".

Just for clarification as you don't mention it specifically, altho your 
btrfs filesystem show information suggests you did it this way, are your 
partition layouts identical on both drives?

That's what I've done here, and I definitely find that easiest to manage 
and even just to think about, tho it's definitely not a requirement.  But 
using different partition layouts does significantly increase management 
complexity, so it's useful to avoid if possible. =:^)

> If I boot on sdb, it takes sda1 as the root filesystem

> If I switched the cable, it always take the first hard drive as
> the root filesystem (now sdb)

That's normal /appearance/, but that /appearance/ doesn't fully reflect 
reality.

The problem is that mount output (and /proc/self/mounts), fstab, etc, 
were designed with single-device filesystems in mind, and multi-device 
btrfs has to be made to fix the existing rules as best it can.

So what's actually happening is that the for a btrfs composed of multiple 
devices, since there's only one "device slot" for the kernel to list 
devices, it only displays the first one it happens to come across, even 
tho the filesystem will normally (unless degraded) require that all 
component devices be available and logically assembled into the 
filesystem before it can be mounted.

When you boot on sdb, naturally, the sdb component of the multi-device 
filesystem that the kernel finds, so it's the one listed, even tho the 
filesystem is actually composed of more devices, not just that one.  When 
you switch the cables, the first one is, at least on your system, always 
the first device component of the filesystem detected, so it's always the 
one occupying the single device slot available for display, even tho the 
filesystem has actually assembled all devices into the complete 
filesystem before mounting.

> If I disconnect /dev/sda, the system doesn't boot with a message saying
> that it hasn't found the UUID:
> 
> Scanning for BTRFS filesystems...
> mount: mounting /dev/disk/by-uuid/c64fca2a-5700-4cca-abac-3a61f2f7486c
> on /root failed: Invalid argument
> 
> Can you tell me what I have done incorrectly ?
> Is it because of UEFI ? If yes I haven't understood how I can correct it
> in a simple way.

As you haven't mentioned it and the grub config below doesn't mention it 
either, I'm almost certain that you're simply not aware of the "degraded" 
mount option, and when/how it should be used.

And if you're not aware of that, chances are you're not aware of the 
btrfs wiki, and the multitude of other very helpful information it has 
available.  I'd suggest you spend some time reading it, as it'll very 
likely save you quite some btrfs administration questions and headaches 
down the road, as you continue to work with btrfs.

Bookmark it and refer to it often! =:^)

https://btrfs.wiki.kernel.org

(Click on the guides and usage information in contents under section 5, 
documentation.)

Here's the mount options page.  Note that the kernel btrfs documentation 
also includes mount options:

https://btrfs.wiki.kernel.org/index.php/Mount_options

$KERNELDIR/Documentation/filesystems/btrfs.txt

You should be able to mount a two-device btrfs raid1 filesystem with only 
a single device with the degraded mount option, tho I believe current 
kernels refuse a read-write mount in that case, so you'll have read-only 
access until you btrfs device add a second device, so it can do normal 
raid1 mode once again.

Specifically, from grub, edit the kernel commandline, setting 
rootflags=degraded.  The kernel rootflags parameter is the method by 
which such mount options are passed.

Meanwhile, since the degraded mount-opt is in fact a no-op if btrfs can 
actually find all components of the filesystem, some people choose to 
simply add degraded to their standard mount options (edit the grub config 
to add it at every boot), so they don't have to worry about it.  However, 
that is NOT RECOMMENDED, as the accepted wisdom is that the failure to 
mount undegraded serves as a warning to the sysadmin that something VERY 
WRONG is happening, and that they need to fix it.  They can then add 
degraded temporarily if they wish, in ordered to get the filesystem to 
mount and thus be able to boot, but adding the option routinely at every 
boot bypasses this important warning, and it's all too likely that an 
admin will thus ignore the problem (or not know about it at all) until 
too late.

Altho if it is indeed true that btrfs will now refuse to mount writable 
if it's degraded like that, that's not such a huge issue either, as the 
read-only mount can serve as the same warning.  Still, I certainly prefer 
the refusal to mount entirely without the degraded option, if indeed the 
filesystem is lacking a component device.  There's nothing quite like 
forcing me to actually type in "rootflags=degraded" to rub my face in the 
reality and gravity of the situation I'm in! =:^)

...

That should answer your immediate question, but do read up on the wiki.  
In addition to much of the FAQ, you'll want to read the sysadmin guide 
page, particularly the raid and data duplication section, and the 
multiple devices page, since they're directly apropos to btrfs multi-
device raid modes.  You'll probably want to read the problem FAQ and 
gotchas pages just for the heads-up as well, and likely at least the raid 
section of the use cases page as well.

Meanwhile, I don't believe it's on the wiki, but it's worth noting my 
experience with btrfs raid1 mode in my pre-deployment tests.  Actually, 
with the (I believe) mandatory read-only mount if raid1 is degraded below 
two devices, this problem's going to be harder to run into than it was in 
my testing several kernels ago, but here's what I found:

What I did was writable-degraded-mount first one of the btrfs raid1 pair, 
then the other (with the other one offline in each case), and change a 
test file with each mount, so that the two copies were different, and 
neither one the same as the original file.  Then I remounted the 
filesystem with both devices once again, to see what would happen.

Based on my previous history with mdraid and how i knew it to behave, I 
expected some note in the log about the two devices having unmatched 
write generation and possibly an automated resync to catch the one back 
up to the other, or alternatively, dropping the one from the mount and 
requiring me to do some sort of manual sync (tho I really didn't know 
what sort of btrfs command I'd use for that, but this was pre-deployment 
testing and I was experimenting with the intent of finding this sort of 
thing out!).

That's *NOT* what I got!

What I got was NO warnings, simply one of the two new versions displayed 
when I catted the file.  I'm not sure if it could have shown me the other 
one such that which one it showed was random, or not, but that I didn't 
get a warning was certainly unsettling to me.

Then I unmounted and unplugged the one with that version of the file, and 
remounted degraded again, to check if the other copy had been silently 
updated.  It was exactly as it had been, so the copies were still 
different.

What I'd do after that today were I redoing this test, would be either a 
scrub or a balance, which would presumably find and correct the 
difference.  However, back then I didn't know enough about what I was 
doing to test that, so I didn't, and I still don't actually know how/
whether the difference would have been detected and corrected, since I 
never did actually test that.


My takeaway from that test was not to actually play around with degraded 
writable mounts to much, and for SURE if I did, to take care that if I 
was to write-mount one and ever intended to bring back the other one, I 
should be sure it was always the same one I was write-mounting and 
updating, so only one would be changed and it'd always be clear which 
copy was the newest.  (Btrfs behavior on this point has since been 
confirmed by a dev, btrfs tracks write generation and will always take 
the higher sequence write generation if there's a difference.  If the 
write generations happened to be the same, however, as I took what he 
said, it'd depend on which one the kernel happened to find first.  So 
always making sure the same one was written to was and remains a good 
idea, so different writes don't get done to different devices, with some 
of those writes dropped when they're recombined in an undegraded mount.)

And if there was any doubt, the best action would be to wipe (or trim/
discard, my devices are SSD so that's the simplest option) the one 
filesystem, and btrfs device add and btrfs balance back to it from the 
other exactly as if it were a new device, rather than risk not knowing 
which of the two differing versions btrfs would end up with.

But as I said, if btrfs only allows read-only mounts of filesystems 
without enough devices to properly complete the raidlevel, that shouldn't 
be as big an issue these days, since it should be more difficult or 
impossible to get the two devices separately mounted writable in the 
first place, with the consequence that the differing copies issue will be 
difficult or impossible to trigger in the first place. =:^)


But that's still a very useful heads-up for anyone using btrfs in raid1 
mode to know about, particularly when they're working with degraded mode, 
just to keep the possibility in mind and be safe with their manipulations 
to avoid it... unless of course they're testing exactly the same sort of 
thing I was. =:^)

> As extra question, I don't see also how I can configure the system to
> get the correct swap in case of disk failure. Should I force both swap
> partition to have the same UUID ?

No.  There's no harm in having multiple swap entries in fstab, so simply 
add a separate fstab entry for each swap partition.  That way, if both 
are available, they'll both be activated by the usual swapon -a.  If only 
one's available, it'll be activated.

(You may want to use the fstab nofail option, described in the fstab (5) 
manpage and mentioned under --ifexists in the swapon (8) manpage as well, 
if your distro's swap initialization doesn't already use --ifexists, to 
prevent a warning if the one doesn't actually exist as it's on a 
disconnected drive.)

As a nice variant, consider using the priority=n option, as detailed in 
the swapon (2,8) manpages.  The kernel defaults to negative priorities, 
with each successive activates swap getting a lower (further negative) 
priority than the others, but you can specify positive swap priorities as 
well.  Higher priority swap is used first, so if you want one swap used 
before another, set its priority higher.

But what REALLY makes swap priorities useful is the fact that if two 
swaps have equal priority, the kernel will automatically effectively 
raid0-stripe swap between them, thus effectively multiplying your swap 
speed! =:^)  Since especially spinning rust is so slow, if your drives 
are spinning rust, that can help significantly if you're actually using 
swap, especially when using it heavily (thrashing).

With ssds the raid0 effect of equal swap priorities should still be 
noticeable, tho they're typically enough faster than spinning rust that 
actively swapping to just one isn't the huge drag it is on spinning rust, 
and with the limited write-cycles of ssd, you may wish to set one a 
higher swap priority than the other just so it gets used more, sparing 
the other.

[Some of the outputs also posted were useful, particularly in implying 
that you had identical partition layout on each device as well as to 
verify that you weren't already using the degraded mount option, but I've 
snipped them as too much to include here.]

-- 
Duncan - List replies preferred.   No HTML msgs.
"Every nonfree program has a lord, a master --
and if you use the program, he is your master."  Richard Stallman

Re: BTRFS with RAID1 cannot boot when removing drive

[Chris Murphy]

On Feb 9, 2014, at 2:40 PM, Saint Germain <saintger@gmail.com> wrote:
> 
> Then I added another drive for a RAID1 configuration (with btrfs
> balance) and I installed grub on the second hard drive with
> "grub-install /dev/sdb".

That can't work on UEFI. UEFI firmware effectively requires a GPT partition map and something to serve as an EFI System partition on all bootable drives.

Second there's a difference between UEFI with and without secure boot.

With secure boot you need to copy the files your distro installer puts on the target drive EFI System partition to each addition drive's ESP if you want multibooting to work in case of disk failure. The grub on each ESP likely looks on only its own ESP for a grub.cfg. So that then means having to sync grub.cfg's among each disk used for booting. A way around this is to create a single grub.cfg that merely forwards to the "true" grub.cfg. And you can copy this forward-only grub.cfg to each ESP. That way the ESP's never need updating or syncing again.

Without secure boot, you must umount /boot/efi and mount the ESP for each bootable disk is turn, and then merely run:

grub-install

That will cause a core.img to be created for that particular ESP, and it will point to the usual grub.cfg location at /boot/grub.



> 
> If I boot on sdb, it takes sda1 as the root filesystem
> If I switched the cable, it always take the first hard drive as the
> root filesystem (now sdb)
> If I disconnect /dev/sda, the system doesn't boot with a message
> saying that it hasn't found the UUID:
> 
> Scanning for BTRFS filesystems...
> mount: mounting /dev/disk/by-uuid/c64fca2a-5700-4cca-abac-3a61f2f7486c on /root failed: Invalid argument

Well if /dev/sda is missing, and you have an unpartitioned /dev/sdb I don't even know how you're getting this far, and it seems like the UEFI computer might actually be booting in CSM-BIOS mode which presents a conventional BIOS to the operating system. Disintguishing such things gets messy quickly.


> 
> Can you tell me what I have done incorrectly ?
> Is it because of UEFI ? If yes I haven't understood how I can correct
> it in a simple way.
> 
> As extra question, I don't see also how I can configure the system to
> get the correct swap in case of disk failure. Should I force both swap partition
> to have the same UUID ?

If you're really expecting to create a system that can accept a disk failure and continue to work, I don't see how it can depend on swap partitions. It's fine to create them, but just realize if they're actually being used and the underlying physical device dies, the kernel isn't going to like it.

A possible work around is using an md raid1 partition as swap.


Chris Murphy

Re: BTRFS with RAID1 cannot boot when removing drive

[Saint Germain]

Hello Duncan,

What an amazing extensive answer you gave me !
Thank you so much for it.

See my comments below.

On Mon, 10 Feb 2014 03:34:49 +0000 (UTC), Duncan <1i5t5.duncan@cox.net>
wrote :

> > I am experimenting with BTRFS and RAID1 on my Debian Wheezy (with
> > backported kernel 3.12-0.bpo.1-amd64) using a a motherboard with
> > UEFI.
> 
> My systems don't do UEFI, but I do run GPT partitions and use grub2
> for booting, with grub2-core installed to a BIOS/reserved type
> partition (instead of as an EFI service as it would be with UEFI).
> And I have root filesystem btrfs two-device raid1 mode working fine
> here, tested bootable with only one device of the two available.
> 
> So while I can't help you directly with UEFI, I know the rest of it
> can/ does work.
> 
> One more thing:  I do have a (small) separate btrfs /boot, actually
> two of them as I setup a separate /boot on each of the two devices in
> ordered to have a backup /boot, since grub can only point to
> one /boot by default, and while pointing to another in grub's rescue
> mode is possible, I didn't want to have to deal with that if the
> first /boot was corrupted, as it's easier to simply point the BIOS at
> a different drive entirely and load its (independently installed and
> configured) grub and /boot.
> 

Can you explain why you choose to have a dedicated "/boot" partition ?
I also read on this thread that it may be better to have a
dedicated /boot partition:
https://bbs.archlinux.org/viewtopic.php?pid=1342893#p1342893


> > However I haven't managed to make the system boot when the removing
> > the first hard drive.
> > 
> > I have installed Debian with the following partition on the first
> > hard drive (no BTRFS subsystem):
> > /dev/sda1: for / (BTRFS)
> > /dev/sda2: for /home (BTRFS)
> > /dev/sda3: for swap
> > 
> > Then I added another drive for a RAID1 configuration (with btrfs
> > balance) and I installed grub on the second hard drive with
> > "grub-install /dev/sdb".
> 
> Just for clarification as you don't mention it specifically, altho
> your btrfs filesystem show information suggests you did it this way,
> are your partition layouts identical on both drives?
> 
> That's what I've done here, and I definitely find that easiest to
> manage and even just to think about, tho it's definitely not a
> requirement.  But using different partition layouts does
> significantly increase management complexity, so it's useful to avoid
> if possible. =:^)

Yes, the partition layout is exactly the same on both drive (copied
with sfdisk). I also try to keep things simple ;-)

> > If I boot on sdb, it takes sda1 as the root filesystem
> 
> > If I switched the cable, it always take the first hard drive as
> > the root filesystem (now sdb)
> 
> That's normal /appearance/, but that /appearance/ doesn't fully
> reflect reality.
> 
> The problem is that mount output (and /proc/self/mounts), fstab, etc, 
> were designed with single-device filesystems in mind, and
> multi-device btrfs has to be made to fix the existing rules as best
> it can.
> 
> So what's actually happening is that the for a btrfs composed of
> multiple devices, since there's only one "device slot" for the kernel
> to list devices, it only displays the first one it happens to come
> across, even tho the filesystem will normally (unless degraded)
> require that all component devices be available and logically
> assembled into the filesystem before it can be mounted.
> 
> When you boot on sdb, naturally, the sdb component of the
> multi-device filesystem that the kernel finds, so it's the one
> listed, even tho the filesystem is actually composed of more devices,
> not just that one.

I am not following you: it seems to be the opposite of what you
describe. If I boot on sdb, I expect sdb1 and sdb2 to be the first
components that the kernel find. However I can see that sda1 and sda2
are used (using the 'mount' command).

> When you switch the cables, the first one is, at
> least on your system, always the first device component of the
> filesystem detected, so it's always the one occupying the single
> device slot available for display, even tho the filesystem has
> actually assembled all devices into the complete filesystem before
> mounting.
> 

Normally the 2 hard drive should be exactly the same (or I didn't
understand something) except for the UUID_SUB.
That's why I don't understand if I switch the cable, I should get
exactly the same results with 'mount'.
But that is not the case, the 'mount' command always point to the same
partition:
- without cable switch: sda1 and sda2
- with cable switch: sdb1 and sdb2
Everything happen as if the system is using the UUID_SUB to get his
'favorite' partition.

> > If I disconnect /dev/sda, the system doesn't boot with a message
> > saying that it hasn't found the UUID:
> > 
> > Scanning for BTRFS filesystems...
> > mount:
> > mounting /dev/disk/by-uuid/c64fca2a-5700-4cca-abac-3a61f2f7486c
> > on /root failed: Invalid argument
> > 
> > Can you tell me what I have done incorrectly ?
> > Is it because of UEFI ? If yes I haven't understood how I can
> > correct it in a simple way.
> 
> As you haven't mentioned it and the grub config below doesn't mention
> it either, I'm almost certain that you're simply not aware of the
> "degraded" mount option, and when/how it should be used.
> 

Ah yes I read about it but didn't understand that it applied in my
sitution. Thanks for pointing it out.

> You should be able to mount a two-device btrfs raid1 filesystem with
> only a single device with the degraded mount option, tho I believe
> current kernels refuse a read-write mount in that case, so you'll
> have read-only access until you btrfs device add a second device, so
> it can do normal raid1 mode once again.
> 

Indeed I managed to boot with the degraded option.
However sda1 is mounted ny default in read-write (not read only) and
sda2 (my /home) refuse to be mounted:
Mounting local filesystemsError mounting: mount: wrong fs type, bad
option, bad superblock on /dev/sda2, missing codepage or helper
program, or other error.
This is with the kernel 3.12-0.bpo.1-amd64.

> That should answer your immediate question, but do read up on the
> wiki. In addition to much of the FAQ, you'll want to read the
> sysadmin guide page, particularly the raid and data duplication
> section, and the multiple devices page, since they're directly
> apropos to btrfs multi- device raid modes.  You'll probably want to
> read the problem FAQ and gotchas pages just for the heads-up as well,
> and likely at least the raid section of the use cases page as well.

Will do !

> 
> Meanwhile, I don't believe it's on the wiki, but it's worth noting my 
> experience with btrfs raid1 mode in my pre-deployment tests.
> Actually, with the (I believe) mandatory read-only mount if raid1 is
> degraded below two devices, this problem's going to be harder to run
> into than it was in my testing several kernels ago, but here's what I
> found:
> 
> What I did was writable-degraded-mount first one of the btrfs raid1
> pair, then the other (with the other one offline in each case), and
> change a test file with each mount, so that the two copies were
> different, and neither one the same as the original file.  Then I
> remounted the filesystem with both devices once again, to see what
> would happen.
> 
> Based on my previous history with mdraid and how i knew it to behave,
> I expected some note in the log about the two devices having
> unmatched write generation and possibly an automated resync to catch
> the one back up to the other, or alternatively, dropping the one from
> the mount and requiring me to do some sort of manual sync (tho I
> really didn't know what sort of btrfs command I'd use for that, but
> this was pre-deployment testing and I was experimenting with the
> intent of finding this sort of thing out!).
> 
> That's *NOT* what I got!
> 
> What I got was NO warnings, simply one of the two new versions
> displayed when I catted the file.  I'm not sure if it could have
> shown me the other one such that which one it showed was random, or
> not, but that I didn't get a warning was certainly unsettling to me.
> 
> Then I unmounted and unplugged the one with that version of the file,
> and remounted degraded again, to check if the other copy had been
> silently updated.  It was exactly as it had been, so the copies were
> still different.
> 
> What I'd do after that today were I redoing this test, would be
> either a scrub or a balance, which would presumably find and correct
> the difference.  However, back then I didn't know enough about what I
> was doing to test that, so I didn't, and I still don't actually know
> how/ whether the difference would have been detected and corrected,
> since I never did actually test that.
> 
> 
> My takeaway from that test was not to actually play around with
> degraded writable mounts to much, and for SURE if I did, to take care
> that if I was to write-mount one and ever intended to bring back the
> other one, I should be sure it was always the same one I was
> write-mounting and updating, so only one would be changed and it'd
> always be clear which copy was the newest.  (Btrfs behavior on this
> point has since been confirmed by a dev, btrfs tracks write
> generation and will always take the higher sequence write generation
> if there's a difference.  If the write generations happened to be the
> same, however, as I took what he said, it'd depend on which one the
> kernel happened to find first.  So always making sure the same one
> was written to was and remains a good idea, so different writes don't
> get done to different devices, with some of those writes dropped when
> they're recombined in an undegraded mount.)
> 
> And if there was any doubt, the best action would be to wipe (or trim/
> discard, my devices are SSD so that's the simplest option) the one 
> filesystem, and btrfs device add and btrfs balance back to it from
> the other exactly as if it were a new device, rather than risk not
> knowing which of the two differing versions btrfs would end up with.
> 
> But as I said, if btrfs only allows read-only mounts of filesystems 
> without enough devices to properly complete the raidlevel, that
> shouldn't be as big an issue these days, since it should be more
> difficult or impossible to get the two devices separately mounted
> writable in the first place, with the consequence that the differing
> copies issue will be difficult or impossible to trigger in the first
> place. =:^)
> 
> 
> But that's still a very useful heads-up for anyone using btrfs in
> raid1 mode to know about, particularly when they're working with
> degraded mode, just to keep the possibility in mind and be safe with
> their manipulations to avoid it... unless of course they're testing
> exactly the same sort of thing I was. =:^)
> 

Well I think I just experienced it the hard way.
I tried unplugging one hard drive and the other to test a little how
BTRFS will react and after several times (I absolutely didn't make
any modification whatsoever) the system refuse to boot.
I tried everything for hours to restore it (btrfsck, scrub, etc.) but I
keep receiving error messages.
At the end I just reinstall everything (fortunately it was a test
system).
So yes you are right, as soon as one hard drive failed, you MUST mount
in read-only mode, otherwise it is almost a given that something bad
will happen.

I think I will try again to experiment, but by taking snapshots before
this time ;-)

Thanks again for your superb help ! It keeps me motivated to keep on
with BTRFS !

Re: BTRFS with RAID1 cannot boot when removing drive

[Saint Germain]

Hello !

On Mon, 10 Feb 2014 19:18:22 -0700, Chris Murphy
<lists@colorremedies.com> wrote :

> 
> On Feb 9, 2014, at 2:40 PM, Saint Germain <saintger@gmail.com> wrote:
> > 
> > Then I added another drive for a RAID1 configuration (with btrfs
> > balance) and I installed grub on the second hard drive with
> > "grub-install /dev/sdb".
> 
> That can't work on UEFI. UEFI firmware effectively requires a GPT
> partition map and something to serve as an EFI System partition on
> all bootable drives.
> 
> Second there's a difference between UEFI with and without secure boot.
> 
> With secure boot you need to copy the files your distro installer
> puts on the target drive EFI System partition to each addition
> drive's ESP if you want multibooting to work in case of disk failure.
> The grub on each ESP likely looks on only its own ESP for a grub.cfg.
> So that then means having to sync grub.cfg's among each disk used for
> booting. A way around this is to create a single grub.cfg that merely
> forwards to the "true" grub.cfg. And you can copy this forward-only
> grub.cfg to each ESP. That way the ESP's never need updating or
> syncing again.
> 
> Without secure boot, you must umount /boot/efi and mount the ESP for
> each bootable disk is turn, and then merely run:
> 
> grub-install
> 
> That will cause a core.img to be created for that particular ESP, and
> it will point to the usual grub.cfg location at /boot/grub.
> 

Ok I need to really understand how my motherboard works (new Z87E-ITX).
It is written "64Mb AMI UEFI Legal BIOS", so I thought it was really
UEFI.

> 
> > 
> > If I boot on sdb, it takes sda1 as the root filesystem
> > If I switched the cable, it always take the first hard drive as the
> > root filesystem (now sdb)
> > If I disconnect /dev/sda, the system doesn't boot with a message
> > saying that it hasn't found the UUID:
> > 
> > Scanning for BTRFS filesystems...
> > mount:
> > mounting /dev/disk/by-uuid/c64fca2a-5700-4cca-abac-3a61f2f7486c
> > on /root failed: Invalid argument
> 
> Well if /dev/sda is missing, and you have an unpartitioned /dev/sdb I
> don't even know how you're getting this far, and it seems like the
> UEFI computer might actually be booting in CSM-BIOS mode which
> presents a conventional BIOS to the operating system. Disintguishing
> such things gets messy quickly.
> 

/dev/sdb has the same partition as /dev/sda.
Duncan gave me the hint with degraded mode and I managed to boot
(however I had some problem with mounting sda2).

> > 
> > Can you tell me what I have done incorrectly ?
> > Is it because of UEFI ? If yes I haven't understood how I can
> > correct it in a simple way.
> > 
> > As extra question, I don't see also how I can configure the system
> > to get the correct swap in case of disk failure. Should I force
> > both swap partition to have the same UUID ?
> 
> If you're really expecting to create a system that can accept a disk
> failure and continue to work, I don't see how it can depend on swap
> partitions. It's fine to create them, but just realize if they're
> actually being used and the underlying physical device dies, the
> kernel isn't going to like it.
> 
> A possible work around is using an md raid1 partition as swap.
> 

I understand. Normally the swap will only be used for hibernating. I
don't expect to use it except perhaps in some extreme case.

Thanks for your help !

Re: BTRFS with RAID1 cannot boot when removing drive

[Duncan]

Saint Germain posted on Tue, 11 Feb 2014 04:15:27 +0100 as excerpted:

> Ok I need to really understand how my motherboard works (new Z87E-ITX).
> It is written "64Mb AMI UEFI Legal BIOS", so I thought it was really
> UEFI.

I expect it's truly UEFI.  But from what I've read most UEFI based 
firmware(possibly all in theory, with the caveat that there's bugs and 
some might not actually work as intended due to bugs) on x86/amd64 (as 
opposed to arm) has a legacy-BIOS mode fallback.  Provided it's not in 
secure-boot mode, if the storage devices it is presented don't have a 
valid UEFI config, it'll fall back to legacy-BIOS mode and try to detect 
and boot that.

Which may or may not be what your system is actually doing.  As I said, 
since I've not actually experimented with UEFI here, my practical 
knowledge on it is virtually nil, and I don't claim to have studied the 
theory well enough to deduce in that level of detail what your system is 
doing.  But I know that's how it's /supposed/ to be able to work. =:^)

(FWIW, what I /have/ done, deliberately, is read enough about UEFI to 
have a general feel for it, and to have been previously exposed to the 
ideas for some time, so that once I /do/ have it available and decide 
it's time, I'll be able to come up to speed relatively quickly as I've 
had the general ideas turning over in my head for quite some time 
already, so in effect I'll simply be reviewing the theory and doing the 
lab work, while concurrently making logical connections about how it all 
fits together that only happen once one actually does that lab work.  
I've discovered over the years that this is perhaps my most effective way 
to learn, read about the general principles while not really 
understanding it the first time thru, then come back to it some months or 
years later, and I pick it up real fast, because my subconscious has been 
working on the problem the whole time! Come to think of it, that's 
actually how I handled btrfs, too, trying it at one point and deciding it 
didn't fit my needs at the time, leaving it for awhile, then coming back 
to it later when my needs had changed, but I already had an idea what I 
was doing from the previous try, with the result being I really took to 
it fast, the second time!  =:^)

-- 
Duncan - List replies preferred.   No HTML msgs.
"Every nonfree program has a lord, a master --
and if you use the program, he is your master."  Richard Stallman

Re: BTRFS with RAID1 cannot boot when removing drive

[Duncan]

Saint Germain posted on Tue, 11 Feb 2014 04:15:27 +0100 as excerpted:

> I understand. Normally the swap will only be used for hibernating. I
> don't expect to use it except perhaps in some extreme case.

If hibernate is your main swap usage, you might consider the noauto fstab 
option as well, then specifically swapon the appropriate one in your 
hibernate script since you may well need logic in there to figure out 
which one to use in any case.  I was doing that for awhile.

(I've run my own suspend/hibernate scripts based on the documentation in 
$KERNDIR/Documentation/power/*, for years.  The kernel's docs dir really 
is a great resource for a lot of sysadmin level stuff as well as the 
expected kernel developer stuff.  I think few are aware of just how much 
real useful admin-level information it actually contains. =:^)

-- 
Duncan - List replies preferred.   No HTML msgs.
"Every nonfree program has a lord, a master --
and if you use the program, he is your master."  Richard Stallman

Re: BTRFS with RAID1 cannot boot when removing drive

[Saint Germain]

On 11 February 2014 07:59, Duncan <1i5t5.duncan@cox.net> wrote:
> Saint Germain posted on Tue, 11 Feb 2014 04:15:27 +0100 as excerpted:
>
>> Ok I need to really understand how my motherboard works (new Z87E-ITX).
>> It is written "64Mb AMI UEFI Legal BIOS", so I thought it was really
>> UEFI.
>
> I expect it's truly UEFI.  But from what I've read most UEFI based
> firmware(possibly all in theory, with the caveat that there's bugs and
> some might not actually work as intended due to bugs) on x86/amd64 (as
> opposed to arm) has a legacy-BIOS mode fallback.  Provided it's not in
> secure-boot mode, if the storage devices it is presented don't have a
> valid UEFI config, it'll fall back to legacy-BIOS mode and try to detect
> and boot that.
>
> Which may or may not be what your system is actually doing.  As I said,
> since I've not actually experimented with UEFI here, my practical
> knowledge on it is virtually nil, and I don't claim to have studied the
> theory well enough to deduce in that level of detail what your system is
> doing.  But I know that's how it's /supposed/ to be able to work. =:^)
>

Hello Duncan;

Yes I also suspect something like that. Unfortunately it is not really
clear on their website how it works.
You can find a lot of marketing stuff, but not what is really needed
to boot properly !

> (FWIW, what I /have/ done, deliberately, is read enough about UEFI to
> have a general feel for it, and to have been previously exposed to the
> ideas for some time, so that once I /do/ have it available and decide
> it's time, I'll be able to come up to speed relatively quickly as I've
> had the general ideas turning over in my head for quite some time
> already, so in effect I'll simply be reviewing the theory and doing the
> lab work, while concurrently making logical connections about how it all
> fits together that only happen once one actually does that lab work.
> I've discovered over the years that this is perhaps my most effective way
> to learn, read about the general principles while not really
> understanding it the first time thru, then come back to it some months or
> years later, and I pick it up real fast, because my subconscious has been
> working on the problem the whole time! Come to think of it, that's
> actually how I handled btrfs, too, trying it at one point and deciding it
> didn't fit my needs at the time, leaving it for awhile, then coming back
> to it later when my needs had changed, but I already had an idea what I
> was doing from the previous try, with the result being I really took to
> it fast, the second time!  =:^)
>

I'll try to keep that in mind !

>> I understand. Normally the swap will only be used for hibernating. I
>> don't expect to use it except perhaps in some extreme case.
>
> If hibernate is your main swap usage, you might consider the noauto fstab
> option as well, then specifically swapon the appropriate one in your
> hibernate script since you may well need logic in there to figure out
> which one to use in any case.  I was doing that for awhile.
>
> (I've run my own suspend/hibernate scripts based on the documentation in
> $KERNDIR/Documentation/power/*, for years.  The kernel's docs dir really
> is a great resource for a lot of sysadmin level stuff as well as the
> expected kernel developer stuff.  I think few are aware of just how much
> real useful admin-level information it actually contains. =:^)

I am not so used to working without swap. I've worked for year with a
computer with low RAM and a swap and I didn't have any problem (even
when doing some RAM intensive task). So I haven't tried to fix it ;-)
If there is sufficient RAM, I suppose that the the swap doesn't get
used so it is not a problem to let it ?
I've hesitated a long time between ZFS and BTRFS, and one of the case
for ZFS was that it can handle natively swap in its "subvolume" (and
so in theory it can enter in the RAID1 as well). However the folks at
ZFS seem also to consider also swap as a relic of the past. I guess I
will keep it just in case. ;-)

The big problem I currently have is that based on your input, I
hesitate a lot on my partitioning scheme: should I use a dedicated
/boot partition or should I have one global BTRFS partition ?
It is not very clear in the doc (a lof of people used a dedicated
/boot because at that time, grub couldn't natively boot BTRFS it
seems, but it has changed).
Could you recommend a partitioning scheme for a simple RAID1 with 2
identical hard drives (just for home computing, not business).

Many thanks !

Re: BTRFS with RAID1 cannot boot when removing drive

[Chris Murphy]

On Feb 10, 2014, at 8:15 PM, Saint Germain <saintger@gmail.com> wrote:
> 
> Ok I need to really understand how my motherboard works (new Z87E-ITX).
> It is written "64Mb AMI UEFI Legal BIOS", so I thought it was really
> UEFI.

Manufacturers have done us a disservice by equating UEFI and BIOS. Some UEFI also have a compatibility support module (CSM) which presents a BIOS to the operating system. It's intended for legacy operating systems that won't ever directly support UEFI.

A way to tell in linux if you're booting with or without the CSM is issue the efibootmgr command. If it return something that looks like an error message, the CSM is enabled and the OS thinks it's running on a BIOS computer. If it returns some numbered list then the CSM isn't enabled and the OS thinks it's running on a UEFI computer.

> /dev/sdb has the same partition as /dev/sda.

grub-install <device> shouldn't work on UEFI because the only place grub-install installs is to the volume mounted at /boot/efi. And also grub-install /dev/sdb implies installing grub to a disk boot sector, which also isn't applicable to UEFI.


> I understand. Normally the swap will only be used for hibernating. I
> don't expect to use it except perhaps in some extreme case.

I consider hibernate fundamentally broken right now because whether it'll work depends on too many things. It works for some people and not others, and for those it does work it largely didn't work out of the box. It never worked for me and did induce Btrfs corruptions so I've just given up on hibernate entirely. There's a long old Fedora thread that discusses myriad issues about it: https://bugzilla.redhat.com/show_bug.cgi?id=781749 and shows even if it's working, it can stop working without warning after X number of hibernate-resume cycles.

Chris Murphy

Re: UEFI/BIOS, was: BTRFS with RAID1 cannot boot when removing drive

[Chris Murphy]

On Feb 10, 2014, at 11:59 PM, Duncan <1i5t5.duncan@cox.net> wrote:

> Saint Germain posted on Tue, 11 Feb 2014 04:15:27 +0100 as excerpted:
> 
>> Ok I need to really understand how my motherboard works (new Z87E-ITX).
>> It is written "64Mb AMI UEFI Legal BIOS", so I thought it was really
>> UEFI.
> 
> I expect it's truly UEFI.  But from what I've read most UEFI based 
> firmware(possibly all in theory, with the caveat that there's bugs and 
> some might not actually work as intended due to bugs) on x86/amd64 (as 
> opposed to arm) has a legacy-BIOS mode fallback.  Provided it's not in 
> secure-boot mode, if the storage devices it is presented don't have a 
> valid UEFI config, it'll fall back to legacy-BIOS mode and try to detect 
> and boot that.

There are UEFI implementations that behave this way with respect to removable and optical devices, they'll try to boot UEFI mode first, and then fallback to BIOS. I've yet to find one that does this for a HDD although maybe it exists. What I've seen is the NVRAM has a boot option that expressly calls for booting a particular device with CSM-BIOS mode boot, or the user has to go into firmware setup and enable the CSM which does so for all boots. This option is sometimes hideously labeled as "disable UEFI". There are some (probably rare) UEFI firmware implementations without a CSM, the only two I can think of off hand are Itanium computers, and Apple's intel rack servers (since discontinued). But CSM isn't a UEFI requirement so there may be other cases where manufacturers have decided to go with only EFI or BIOS.

Chris Murphy

Re: BTRFS with RAID1 cannot boot when removing drive

[Saint Germain]

On 11 February 2014 18:21, Chris Murphy <lists@colorremedies.com> wrote:
>
> On Feb 10, 2014, at 8:15 PM, Saint Germain <saintger@gmail.com> wrote:
>>
>> Ok I need to really understand how my motherboard works (new Z87E-ITX).
>> It is written "64Mb AMI UEFI Legal BIOS", so I thought it was really
>> UEFI.
>
> Manufacturers have done us a disservice by equating UEFI and BIOS. Some UEFI also have a compatibility support module (CSM) which presents a BIOS to the operating system. It's intended for legacy operating systems that won't ever directly support UEFI.
>
> A way to tell in linux if you're booting with or without the CSM is issue the efibootmgr command. If it return something that looks like an error message, the CSM is enabled and the OS thinks it's running on a BIOS computer. If it returns some numbered list then the CSM isn't enabled and the OS thinks it's running on a UEFI computer.
>

Nice trick ! Thanks.

>> /dev/sdb has the same partition as /dev/sda.
>
> grub-install <device> shouldn't work on UEFI because the only place grub-install installs is to the volume mounted at /boot/efi. And also grub-install /dev/sdb implies installing grub to a disk boot sector, which also isn't applicable to UEFI.
>

I am still not up to date on UEFI partition and so.
But I have read these pages:
http://tanguy.ortolo.eu/blog/article51/debian-efi
http://forums.debian.net/viewtopic.php?f=16&t=81120
And apparently is grub-install <device> is the correct command (but
you have to copy file in addition).
It is maybe because they use a special package grub-efi-amd64, which
replace the grub-install.
It is quite difficult to find reliable info on the topic...

>
>> I understand. Normally the swap will only be used for hibernating. I
>> don't expect to use it except perhaps in some extreme case.
>
> I consider hibernate fundamentally broken right now because whether it'll work depends on too many things. It works for some people and not others, and for those it does work it largely didn't work out of the box. It never worked for me and did induce Btrfs corruptions so I've just given up on hibernate entirely. There's a long old Fedora thread that discusses myriad issues about it: https://bugzilla.redhat.com/show_bug.cgi?id=781749 and shows even if it's working, it can stop working without warning after X number of hibernate-resume cycles.
>

I am among the fortunate to have a working hibernate out of the box
(Debian stable) and it works reliably (even it ultimately it WILL fail
after 20-30 iterations). So I use the feature to save on electricity
cost ;-)
But yes, maybe I will get rid of the swap...

Thanks !

Re: BTRFS with RAID1 cannot boot when removing drive

[Chris Murphy]

On Feb 11, 2014, at 10:36 AM, Saint Germain <saintger@gmail.com> wrote:
>> 
>> grub-install <device> shouldn't work on UEFI because the only place grub-install installs is to the volume mounted at /boot/efi. And also grub-install /dev/sdb implies installing grub to a disk boot sector, which also isn't applicable to UEFI.
>> 
> 
> I am still not up to date on UEFI partition and so.
> But I have read these pages:
> http://tanguy.ortolo.eu/blog/article51/debian-efi
> http://forums.debian.net/viewtopic.php?f=16&t=81120
> And apparently is grub-install <device> is the correct command (but
> you have to copy file in addition).
> It is maybe because they use a special package grub-efi-amd64, which
> replace the grub-install.
> It is quite difficult to find reliable info on the topic…

grub-install <device> probably works insofar as <device> is being ignored. I bet you get the same results no matter what device you choose, but maybe I'm mistaken and they have some way to write to an unmounted ESP which seems like not such a great idea.


Chris Murphy

Re: BTRFS with RAID1 cannot boot when removing drive

[Duncan]

Saint Germain posted on Tue, 11 Feb 2014 11:04:57 +0100 as excerpted:

> The big problem I currently have is that based on your input, I hesitate
> a lot on my partitioning scheme: should I use a dedicated /boot
> partition or should I have one global BTRFS partition ?
> It is not very clear in the doc (a lof of people used a dedicated /boot
> because at that time, grub couldn't natively boot BTRFS it seems, but it
> has changed).
> Could you recommend a partitioning scheme for a simple RAID1 with 2
> identical hard drives (just for home computing, not business).

FWIW... I'm planning to and have your previous message covering that 
still marked unread to reply to later.  But "real life" has temporarily 
been monopolizing my time so the last day or two I've only done 
relatively short and quick replies.  That one will require a bit more 
time to answer to my satisfaction.

So I'm punting for the moment.  But FWIW I tend to be a reasonably heavy 
partitioner (tho nowhere near what I used to be), so a lot of folks will 
consider my setup somewhat extreme.  That's OK.  It's my computer, setup 
for my purposes, not their computer for theirs, and it works very well 
for me, so it's all good. =:^)

But hopefully I'll get back to that with a longer reply by the end of the 
week.  If I don't, you can probably consider that monopoly lasting longer 
than I thought, and it could be that I'll never get back to properly 
reply.  But it's an interesting enough topic to me that I'll /probably/ 
get back, just not right ATM.

-- 
Duncan - List replies preferred.   No HTML msgs.
"Every nonfree program has a lord, a master --
and if you use the program, he is your master."  Richard Stallman

Re: BTRFS with RAID1 cannot boot when removing drive

[Saint Germain]

On 11 February 2014 21:35, Duncan <1i5t5.duncan@cox.net> wrote:
> Saint Germain posted on Tue, 11 Feb 2014 11:04:57 +0100 as excerpted:
>
>> The big problem I currently have is that based on your input, I hesitate
>> a lot on my partitioning scheme: should I use a dedicated /boot
>> partition or should I have one global BTRFS partition ?
>> It is not very clear in the doc (a lof of people used a dedicated /boot
>> because at that time, grub couldn't natively boot BTRFS it seems, but it
>> has changed).
>> Could you recommend a partitioning scheme for a simple RAID1 with 2
>> identical hard drives (just for home computing, not business).
>
> FWIW... I'm planning to and have your previous message covering that
> still marked unread to reply to later.  But "real life" has temporarily
> been monopolizing my time so the last day or two I've only done
> relatively short and quick replies.  That one will require a bit more
> time to answer to my satisfaction.
>
> So I'm punting for the moment.  But FWIW I tend to be a reasonably heavy
> partitioner (tho nowhere near what I used to be), so a lot of folks will
> consider my setup somewhat extreme.  That's OK.  It's my computer, setup
> for my purposes, not their computer for theirs, and it works very well
> for me, so it's all good. =:^)
>
> But hopefully I'll get back to that with a longer reply by the end of the
> week.  If I don't, you can probably consider that monopoly lasting longer
> than I thought, and it could be that I'll never get back to properly
> reply.  But it's an interesting enough topic to me that I'll /probably/
> get back, just not right ATM.
>

No problem, I have started another thread where we discuss partitioning.
It may be slightly off-topic, but the intention is really to have a
partition BTRFS-friendly.
For instance it seems that a dedicated /boot partition instead of a
subvolume is not necessary (better to have the /boot in the RAID1).
However I'm no expert.

Thanks for your help.

Re: BTRFS with RAID1 cannot boot when removing drive

[Saint Germain]

On 11 February 2014 03:30, Saint Germain <saintger@gmail.com> wrote:
>> > I am experimenting with BTRFS and RAID1 on my Debian Wheezy (with
>> > backported kernel 3.12-0.bpo.1-amd64) using a a motherboard with
>> > UEFI.
>>
>> > I have installed Debian with the following partition on the first
>> > hard drive (no BTRFS subsystem):
>> > /dev/sda1: for / (BTRFS)
>> > /dev/sda2: for /home (BTRFS)
>> > /dev/sda3: for swap
>> >
>> > Then I added another drive for a RAID1 configuration (with btrfs
>> > balance) and I installed grub on the second hard drive with
>> > "grub-install /dev/sdb".
>>
>> You should be able to mount a two-device btrfs raid1 filesystem with
>> only a single device with the degraded mount option, tho I believe
>> current kernels refuse a read-write mount in that case, so you'll
>> have read-only access until you btrfs device add a second device, so
>> it can do normal raid1 mode once again.
>>
>> Meanwhile, I don't believe it's on the wiki, but it's worth noting my
>> experience with btrfs raid1 mode in my pre-deployment tests.
>> Actually, with the (I believe) mandatory read-only mount if raid1 is
>> degraded below two devices, this problem's going to be harder to run
>> into than it was in my testing several kernels ago, but here's what I
>> found:
>>
>> But as I said, if btrfs only allows read-only mounts of filesystems
>> without enough devices to properly complete the raidlevel, that
>> shouldn't be as big an issue these days, since it should be more
>> difficult or impossible to get the two devices separately mounted
>> writable in the first place, with the consequence that the differing
>> copies issue will be difficult or impossible to trigger in the first
>> place. =:^)
>>

Hello,

With your advices and Chris ones, I have now a (clean ?) partition to
start experimenting with RAID1 (and which boot correctly in UEFI
mode):
sda1 = BIOS Boot partition
sda2 = EFI System Partition
sda3 = BTFS partition
sda4 = swap partition
For the moment I haven't created subvolumes (for "/" and for "/home"
for instance) to keep things simple.

The idea is then to create a RAID1 with a sdb drive (duplicate sda
partitioning, add/balance/convert sdb3 + grub-install on sdb, add sdb
swap UUID in /etc/fstab), shutdown and remove sda to check the
procedure to replace it.

I read the last thread on the subject "lost with degraded RAID1", but
would like to really confirm what would be the current approved
procedure and if it will be valid for future BTRFS version (especially
about the read-only mount).

So what should I do from there ?
Here are a few questions:

1) Boot in degraded mode: currently with my kernel
(3.12-0.bpo.1-amd64, from Debian wheezy-backports) it seems that I can
mount in read-write mode.
However for future kernel, it seems that I will be only able to mount
read-only ? See here:
http://www.spinics.net/lists/linux-btrfs/msg20164.html
https://bugzilla.kernel.org/show_bug.cgi?id=60594

2) If I am able to mount read-write, is this the correct procedure:
  a) place a new drive in another physical location sdc (I don't think
I can use the same sda physical location ?)
  b) boot in degraded mode on sdb
  c) use the 'replace' command to replace sda by sdc
  d) perhaps a 'balance' is necessary ?

3) Can I use also the above procedure if I am only allowed to mount read-only ?

4) If I want to use my system without RAID1 support (dangerous I
know), after booting in degraded mode with read-write, can I convert
back sdb from RAID1 to RAID0 in a safe way ?
(btrfs balance start -dconvert=raid0 -mconvert=raid0 /)

5) Perhaps a recovery procedure which includes booting on a different
rescue disk would be more appropriate ?

Thanks again,

Re: BTRFS with RAID1 cannot boot when removing drive

[Saint Germain]

On Fri, 14 Feb 2014 15:33:10 +0100, Saint Germain <saintger@gmail.com>
wrote :

> On 11 February 2014 03:30, Saint Germain <saintger@gmail.com> wrote:
> >> > I am experimenting with BTRFS and RAID1 on my Debian Wheezy (with
> >> > backported kernel 3.12-0.bpo.1-amd64) using a a motherboard with
> >> > UEFI.
> >>
> >> > I have installed Debian with the following partition on the first
> >> > hard drive (no BTRFS subsystem):
> >> > /dev/sda1: for / (BTRFS)
> >> > /dev/sda2: for /home (BTRFS)
> >> > /dev/sda3: for swap
> >> >
> >> > Then I added another drive for a RAID1 configuration (with btrfs
> >> > balance) and I installed grub on the second hard drive with
> >> > "grub-install /dev/sdb".
> >>
> >> You should be able to mount a two-device btrfs raid1 filesystem
> >> with only a single device with the degraded mount option, tho I
> >> believe current kernels refuse a read-write mount in that case, so
> >> you'll have read-only access until you btrfs device add a second
> >> device, so it can do normal raid1 mode once again.
> >>
> >> Meanwhile, I don't believe it's on the wiki, but it's worth noting
> >> my experience with btrfs raid1 mode in my pre-deployment tests.
> >> Actually, with the (I believe) mandatory read-only mount if raid1
> >> is degraded below two devices, this problem's going to be harder
> >> to run into than it was in my testing several kernels ago, but
> >> here's what I found:
> >>
> >> But as I said, if btrfs only allows read-only mounts of filesystems
> >> without enough devices to properly complete the raidlevel, that
> >> shouldn't be as big an issue these days, since it should be more
> >> difficult or impossible to get the two devices separately mounted
> >> writable in the first place, with the consequence that the
> >> differing copies issue will be difficult or impossible to trigger
> >> in the first place. =:^)
> >>
> 
> Hello,
> 
> With your advices and Chris ones, I have now a (clean ?) partition to
> start experimenting with RAID1 (and which boot correctly in UEFI
> mode):
> sda1 = BIOS Boot partition
> sda2 = EFI System Partition
> sda3 = BTFS partition
> sda4 = swap partition
> For the moment I haven't created subvolumes (for "/" and for "/home"
> for instance) to keep things simple.
> 
> The idea is then to create a RAID1 with a sdb drive (duplicate sda
> partitioning, add/balance/convert sdb3 + grub-install on sdb, add sdb
> swap UUID in /etc/fstab), shutdown and remove sda to check the
> procedure to replace it.
> 
> I read the last thread on the subject "lost with degraded RAID1", but
> would like to really confirm what would be the current approved
> procedure and if it will be valid for future BTRFS version (especially
> about the read-only mount).
> 
> So what should I do from there ?
> Here are a few questions:
> 
> 1) Boot in degraded mode: currently with my kernel
> (3.12-0.bpo.1-amd64, from Debian wheezy-backports) it seems that I can
> mount in read-write mode.
> However for future kernel, it seems that I will be only able to mount
> read-only ? See here:
> http://www.spinics.net/lists/linux-btrfs/msg20164.html
> https://bugzilla.kernel.org/show_bug.cgi?id=60594
> 
> 2) If I am able to mount read-write, is this the correct procedure:
>   a) place a new drive in another physical location sdc (I don't think
> I can use the same sda physical location ?)
>   b) boot in degraded mode on sdb
>   c) use the 'replace' command to replace sda by sdc
>   d) perhaps a 'balance' is necessary ?
> 
> 3) Can I use also the above procedure if I am only allowed to mount
> read-only ?
> 
> 4) If I want to use my system without RAID1 support (dangerous I
> know), after booting in degraded mode with read-write, can I convert
> back sdb from RAID1 to RAID0 in a safe way ?
> (btrfs balance start -dconvert=raid0 -mconvert=raid0 /)
> 

To continue with this RAID1 recovery procedure (Debian stable with
kernel 3.12-0.bpo.1-amd64), I tried to reproduce Duncan setup and the
result is not good.

Starting with a clean setup of 2 hard drive in RAID1 (sda and sdb) and
a clean snapshot of the rootfs:
1) poweroff, disconnect sda and boot on sdb with rootflags=ro,degraded
2) sdb is mounted ro but automatically remounted read-write by initramf
3) create a file witness1 and modify a file test.txt with 'alpha' inside
4) poweroff, connect sda, disconnect sdb and boot on sda
5) create a file witness2 and modify a file test.txt with 'beta' inside
6) poweroff, connect sdb and boot on sda
7) the modification from step 3 are there (but not from step 5)
8) launch scrub: a lot of errors are detected but no unrepairable errors
9) poweroff, disconnect sdb, boot on sda
10) the modification from step 3 are there (but not from step 5)
11) poweroff, boot on sda: kernel panic on startup
12) reboot, boot is possible
13) launch scrub: a lot of errors and kernel error
14) reboot, error on boot, and same error as step 13 with scrub
15) boot on previous snapshot of step1, same error on boot and
same error as step 13 with scrub.


I hope that it will be useful for someone. It seems that mounting
read-write is really not a good idea (have to find how to force ro with
Debian). The RAID1 configuration is a bit fagile and even snapshots
won't protect you when it is broken.

If someone has some insigths, please let me know.

Here is for info the kernel error at boot and the scrub error:
[   37.575270] BUG: unable to handle kernel NULL pointer dereference at 00000000000000c0
[   37.575299] IP: [<ffffffffa01a8a3b>] __btrfs_cow_block+0x28b/0x540 [btrfs]
[   37.575328] PGD 0 
[   37.575337] Oops: 0000 [#1] SMP 
[   37.575351] Modules linked in: cpufreq_userspace cpufreq_stats cpufreq_powersave cpufreq_conservative nfsd auth_rpcgss oid_registry nfs_acl nfs lockd fscache sunrpc nls_utf8 nls_cp437 vfat fat loop x86_pkg_temp_thermal coretemp kvm_intel snd_hda_codec_hdmi snd_hda_codec_realtek kvm snd_hda_intel snd_hda_codec lib80211_crypt_tkip snd_hwdep crct10dif_pclmul crc32_pclmul crc32c_intel ghash_clmulni_intel aesni_intel wl(PO) snd_pcm aes_x86_64 lrw gf128mul glue_helper snd_page_alloc ablk_helper snd_timer iTCO_wdt iTCO_vendor_support cryptd psmouse acpi_cpufreq snd cfg80211 lib80211 i915 serio_raw drm_kms_helper drm processor button video thermal_sys joydev evdev rfkill i2c_i801 i2c_algo_bit lpc_ich i2c_core pcspkr mei_me mei mfd_core soundcore btrfs xor hid_generic usbhid hid raid6_pq crc32c libcrc32c sg sd_mod sr_mod cdrom crc_t10dif crct10dif_common ahci libahci libata scsi_mod xhci_hcd ehci_pci ehci_hcd e1000e ptp pps_core usbcore usb_common
[   37.575667] CPU: 0 PID: 297 Comm: btrfs-transacti Tainted: P           O 3.12-0.bpo.1-amd64 #1 Debian 3.12.9-1~bpo70+1
[   37.575695] Hardware name: To Be Filled By O.E.M. To Be Filled By O.E.M./Z87E-ITX, BIOS P2.10 10/04/2013
[   37.575719] task: ffff8800610747c0 ti: ffff880061060000 task.ti: ffff880061060000
[   37.575738] RIP: 0010:[<ffffffffa01a8a3b>]  [<ffffffffa01a8a3b>] __btrfs_cow_block+0x28b/0x540 [btrfs]
[   37.575766] RSP: 0018:ffff8800610618e8  EFLAGS: 00010217
[   37.575781] RAX: 0000160000000000 RBX: ffff88006124f800 RCX: ffff880060f92ea8
[   37.575799] RDX: ffff880000000000 RSI: 000000000000027b RDI: ffff880064a79270
[   37.575818] RBP: ffff880064948920 R08: ffff880061061944 R09: 0000000000000000
[   37.575837] R10: ffff88003734a000 R11: 0000000000000000 R12: ffff880060fe5960
[   37.575856] R13: ffff880064a79270 R14: 0000000000000000 R15: 0000000000000000
[   37.575874] FS:  0000000000000000(0000) GS:ffff880100200000(0000) knlGS:0000000000000000
[   37.575895] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[   37.575910] CR2: 00000000000000c0 CR3: 000000000180c000 CR4: 00000000001407f0
[   37.575928] Stack:
[   37.575935]  ffff880000000000 0000000300000000 0000000000000000 ffffffffa01cb08d
[   37.575959]  ffff880061061a38 0000000000000000 ffff880064948920 0000000000000000
[   37.575983]  0000000000001000 000000b8a21aab99 0000000001000000 0000000100000000
[   37.576007] Call Trace:
[   37.576022]  [<ffffffffa01cb08d>] ? btrfs_buffer_uptodate+0x6d/0x80 [btrfs]
[   37.576044]  [<ffffffffa01a8ecb>] ? btrfs_cow_block+0x13b/0x200 [btrfs]
[   37.576069]  [<ffffffffa02044e2>] ? btrfs_set_lock_blocking_rw+0xa2/0xe0 [btrfs]
[   37.576092]  [<ffffffffa01ad33f>] ? btrfs_search_slot+0x46f/0x9c0 [btrfs]
[   37.576115]  [<ffffffffa01b365d>] ? lookup_inline_extent_backref+0xcd/0x5e0 [btrfs]
[   37.576139]  [<ffffffffa01b3be4>] ? insert_inline_extent_backref+0x74/0x120 [btrfs]
[   37.576163]  [<ffffffffa01b5a7f>] ? update_block_group.isra.75+0xbf/0x270 [btrfs]
[   37.576183]  [<ffffffff8116f97c>] ? kmem_cache_alloc+0x1bc/0x1f0
[   37.576203]  [<ffffffffa01b44ef>] ? __btrfs_inc_extent_ref+0xaf/0x250 [btrfs]
[   37.576227]  [<ffffffffa01bb47b>] ? run_clustered_refs+0xd4b/0xef0 [btrfs]
[   37.576251]  [<ffffffffa0214804>] ? btrfs_find_ref_cluster+0x74/0x170 [btrfs]
[   37.576274]  [<ffffffffa01bf370>] ? btrfs_run_delayed_refs+0xd0/0x530 [btrfs]
[   37.576300]  [<ffffffffa01e8683>] ? btrfs_run_ordered_operations+0x213/0x2b0 [btrfs]
[   37.576326]  [<ffffffffa01cf90a>] ? btrfs_commit_transaction+0x5a/0x9f0 [btrfs]
[   37.576350]  [<ffffffffa01c9385>] ? transaction_kthread+0x1b5/0x220 [btrfs]
[   37.576373]  [<ffffffffa01c91d0>] ? btree_readpage_end_io_hook+0x2d0/0x2d0 [btrfs]
[   37.576394]  [<ffffffff81082333>] ? kthread+0xb3/0xc0
[   37.576408]  [<ffffffff81082280>] ? flush_kthread_worker+0xa0/0xa0
[   37.576426]  [<ffffffff814cb70c>] ? ret_from_fork+0x7c/0xb0
[   37.576442]  [<ffffffff81082280>] ? flush_kthread_worker+0xa0/0xa0
[   37.576458] Code: 00 48 39 2b 0f 84 e6 01 00 00 48 83 bb d7 01 00 00 f8 48 c7 44 24 38 00 00 00 00 0f 84 bf 01 00 00 48 b8 00 00 00 00 00 16 00 00 <49> 03 87 c0 00 00 00 48 ba b7 6d db b6 6d db b6 6d 48 c1 f8 03 
[   37.576578] RIP  [<ffffffffa01a8a3b>] __btrfs_cow_block+0x28b/0x540 [btrfs]
[   37.576601]  RSP <ffff8800610618e8>
[   37.576611] CR2: 00000000000000c0
[   37.576622] ---[ end trace eb7650bbae358a3a ]---

And here is for info the scrub error:
[  390.015858] BTRFS critical (device sdb3): unable to find logical 687194767360 len 4096
[  390.015957] ------------[ cut here ]------------
[  390.015989] kernel BUG at /build/linux-SMWX37/linux-3.12.9/fs/btrfs/inode.c:1595!
[  390.016037] invalid opcode: 0000 [#1] SMP 
[  390.016070] Modules linked in: cpufreq_userspace cpufreq_stats cpufreq_powersave cpufreq_conservative nfsd auth_rpcgss oid_registry nfs_acl nfs lockd fscache sunrpc nls_utf8 nls_cp437 vfat fat loop x86_pkg_temp_thermal coretemp kvm_intel kvm crct10dif_pclmul crc32_pclmul crc32c_intel ghash_clmulni_intel aesni_intel aes_x86_64 snd_hda_codec_hdmi snd_hda_codec_realtek lrw gf128mul snd_hda_intel snd_hda_codec snd_hwdep lib80211_crypt_tkip wl(PO) iTCO_wdt iTCO_vendor_support glue_helper ablk_helper snd_pcm cryptd acpi_cpufreq pcspkr psmouse mei_me mei i915 lpc_ich video processor drm_kms_helper drm mfd_core i2c_algo_bit snd_page_alloc snd_timer snd cfg80211 lib80211 rfkill i2c_i801 soundcore thermal_sys serio_raw i2c_core joydev evdev button hid_generic usbhid hid btrfs xor raid6_pq crc32c libcrc32c sg sd_mod sr_mod cdrom crc_t10dif crct10dif_common ahci libahci xhci_hcd ehci_pci ehci_hcd e1000e ptp pps_core libata scsi_mod usbcore usb_common
[  390.016840] CPU: 0 PID: 3045 Comm: btrfs Tainted: P           O 3.12-0.bpo.1-amd64 #1 Debian 3.12.9-1~bpo70+1
[  390.016899] Hardware name: To Be Filled By O.E.M. To Be Filled By O.E.M./Z87E-ITX, BIOS P2.10 10/04/2013
[  390.016957] task: ffff88006a9207c0 ti: ffff8800641be000 task.ti: ffff8800641be000
[  390.017004] RIP: 0010:[<ffffffffa01b5671>]  [<ffffffffa01b5671>] btrfs_merge_bio_hook+0x71/0x80 [btrfs]
[  390.017084] RSP: 0018:ffff8800641bf4f8  EFLAGS: 00010282
[  390.017119] RAX: 00000000ffffffea RBX: 0000000000001000 RCX: 0000000000000000
[  390.017163] RDX: ffff88010020ffa8 RSI: ffff88010020e4b8 RDI: 0000000000000246
[  390.017207] RBP: 0000000000001000 R08: 0000000000000000 R09: ffff8800606c5970
[  390.017250] R10: 00000000000002fb R11: ffffffff8164e348 R12: ffff880063c08c28
[  390.017294] R13: 0000000000000000 R14: ffff880064bd88b0 R15: 0000000050000008
[  390.017338] FS:  00007fe25f8fe700(0000) GS:ffff880100200000(0000) knlGS:0000000000000000
[  390.017388] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[  390.017424] CR2: 00007fe25f933f80 CR3: 0000000036435000 CR4: 00000000001407f0
[  390.017493] Stack:
[  390.017518]  0000000000001000 0000000000001000 ffff880064bd88b0 ffff8800641bf768
[  390.017616]  0000000000001000 ffffffffa01cf313 0000000000000000 ffffea000159bff0
[  390.017712]  0000000000000000 ffff880100240380 ffffffffa01cf700 0000000000000020
[  390.017809] Call Trace:
[  390.017867]  [<ffffffffa01cf313>] ? submit_extent_page.isra.38+0xf3/0x250 [btrfs]
[  390.017967]  [<ffffffffa01cf700>] ? repair_io_failure+0x240/0x240 [btrfs]
[  390.018039]  [<ffffffffa01d0587>] ? __do_readpage+0x4b7/0x720 [btrfs]
[  390.018092]  [<ffffffffa01cf700>] ? repair_io_failure+0x240/0x240 [btrfs]
[  390.018150]  [<ffffffffa01a99d0>] ? verify_parent_transid+0x190/0x190 [btrfs]
[  390.018207]  [<ffffffffa01d08a4>] ? __extent_read_full_page+0xb4/0xd0 [btrfs]
[  390.018264]  [<ffffffffa01a99d0>] ? verify_parent_transid+0x190/0x190 [btrfs]
[  390.018320]  [<ffffffffa01a99d0>] ? verify_parent_transid+0x190/0x190 [btrfs]
[  390.018380]  [<ffffffffa01d312d>] ? read_extent_buffer_pages+0x25d/0x350 [btrfs]
[  390.018437]  [<ffffffffa01a99d0>] ? verify_parent_transid+0x190/0x190 [btrfs]
[  390.018493]  [<ffffffffa01aaf59>] ? btree_read_extent_buffer_pages.constprop.128+0xa9/0x110 [btrfs]
[  390.018559]  [<ffffffffa01ad5d3>] ? read_tree_block+0x33/0x60 [btrfs]
[  390.018610]  [<ffffffffa018dc65>] ? read_block_for_search.isra.45+0x195/0x3d0 [btrfs]
[  390.018669]  [<ffffffffa01938b9>] ? btrfs_next_old_leaf+0x2f9/0x490 [btrfs]
[  390.018726]  [<ffffffffa0206056>] ? scrub_stripe+0x5f6/0x1200 [btrfs]
[  390.018771]  [<ffffffff81082d00>] ? add_wait_queue+0x40/0x60
[  390.018820]  [<ffffffffa0206d8c>] ? scrub_chunk.isra.18+0x12c/0x150 [btrfs]
[  390.018875]  [<ffffffffa0207013>] ? scrub_enumerate_chunks+0x263/0x5a0 [btrfs]
[  390.018922]  [<ffffffff81082d00>] ? add_wait_queue+0x40/0x60
[  390.018971]  [<ffffffffa0208756>] ? btrfs_scrub_dev+0x1e6/0x570 [btrfs]
[  390.019027]  [<ffffffffa01e5e31>] ? btrfs_ioctl+0xe91/0x1d30 [btrfs]
[  390.019079]  [<ffffffffa01e5e8b>] ? btrfs_ioctl+0xeeb/0x1d30 [btrfs]
[  390.019123]  [<ffffffff814c305d>] ? rwsem_down_read_failed+0x9d/0xf0
[  390.019165]  [<ffffffff8128ef54>] ? call_rwsem_down_read_failed+0x14/0x30
[  390.019210]  [<ffffffff814c75b8>] ? __do_page_fault+0x2b8/0x540
[  390.019252]  [<ffffffff811971ca>] ? do_vfs_ioctl+0x8a/0x4f0
[  390.019289]  [<ffffffff81062c1d>] ? do_exit+0x6fd/0xa80
[  390.019325]  [<ffffffff810135f1>] ? __switch_to+0x171/0x4c0
[  390.019363]  [<ffffffff811976d0>] ? SyS_ioctl+0xa0/0xc0
[  390.019399]  [<ffffffff814cb7b9>] ? system_call_fastpath+0x16/0x1b
[  390.019437] Code: c9 45 31 c0 89 fe 48 89 c7 48 89 6c 24 08 e8 97 5f 02 00 85 c0 78 14 48 01 eb 31 c0 48 3b 5c 24 08 0f 97 c0 48 83 c4 18 5b 5d c3 <0f> 0b 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 53 48 89 fb 89 f1 
[  390.019733] RIP  [<ffffffffa01b5671>] btrfs_merge_bio_hook+0x71/0x80 [btrfs]
[  390.019793]  RSP <ffff8800641bf4f8>
[  390.019838] ---[ end trace 74720f4e8a3bc0fa ]---