[BUG] raw over raid5: BUG at drivers/block/ll_rw_blk.c:1967

[BUG] raw over raid5: BUG at drivers/block/ll_rw_blk.c:1967

[David Mansfield]

Hi everyone,

I haven't been able to run raw over raid5 since 2.5.30 or so, but every
time I'm about to report it, a new kernel comes out and the problem
changes completely :-( Now I'm finally going to start getting out the info
it the hopes someone can fix it.  The oops was triggered by attempting to 
read from /dev/raw/raw1 (bound to /dev/md0) using dd.  System info 
follows oops:

------------[ cut here ]------------
kernel BUG at drivers/block/ll_rw_blk.c:1967!
invalid operand: 0000
 
CPU:    0
EIP:    0060:[<c024ecc8>]    Not tainted
EFLAGS: 00010206
EIP is at end_that_request_first+0x98/0x220
eax: 00001000   ebx: 00000008   ecx: 00000000   edx: 00000000
esi: f6d3e9e0   edi: f6d3ea1c   ebp: f7d38b40   esp: c038be68
ds: 0068   es: 0068   ss: 0068
Process swapper (pid: 0, threadinfo=c038a000 task=c0369d00)
Stack: 00000000 00000000 00000000 00000001 c038a000 c038bea8 c0117fa0 f7d52400 
       f7d38b40 f7d52400 f7d47818 c0282cb7 f7d38b40 00000001 00000008 f6c14520 
       f7d52400 00000000 00000000 00000000 c0282f78 f7d52400 00000001 00000008 
Call Trace:
 [<c0117fa0>] try_to_wake_up+0x250/0x260
 [<c0282cb7>] __scsi_end_request+0x27/0xd0
 [<c0282f78>] scsi_io_completion+0x188/0x3a0
 [<c0118f1c>] __wake_up+0x2c/0x50
 [<c0299ed3>] sd_rw_intr+0x1f3/0x200
 [<c027d881>] scsi_finish_command+0x81/0x90
 [<c027d6f6>] scsi_softirq+0x86/0x100
 [<c012250b>] do_softirq+0x5b/0xc0
 [<c010b0d0>] do_IRQ+0x140/0x160
 [<c0106ee0>] default_idle+0x0/0x40
 [<c0106ee0>] default_idle+0x0/0x40
 [<c01098f4>] common_interrupt+0x18/0x20
 [<c0106ee0>] default_idle+0x0/0x40
 [<c0106ee0>] default_idle+0x0/0x40
 [<c0106f0a>] default_idle+0x2a/0x40
 [<c0106fb2>] cpu_idle+0x52/0x70
 [<c0105000>] stext+0x0/0x60

Code: 0f 0b af 07 1f 53 33 c0 3b 5c 24 38 7f 61 89 da c1 e2 09 39 
 <0>Kernel panic: Aiee, killing interrupt handler!
In interrupt handler - not syncing

My sytem is a dual 450 MHZ PII with 1GB ram (HIGHMEM4G).  Raid is sw raid5
on 5 SCSI disks running off of an adaptec aic7xxx controller.  Compiler is
"gcc version 2.96 20000731 (Red Hat Linux 7.3 2.96-112)" with an updated
redhat 7.3 toolchain.

Config is:

CONFIG_X86=y
CONFIG_UID16=y
CONFIG_GENERIC_ISA_DMA=y
CONFIG_EXPERIMENTAL=y
CONFIG_NET=y
CONFIG_SYSVIPC=y
CONFIG_SYSCTL=y
CONFIG_MODULES=y
CONFIG_KMOD=y
CONFIG_M686=y
CONFIG_X86_WP_WORKS_OK=y
CONFIG_X86_INVLPG=y
CONFIG_X86_CMPXCHG=y
CONFIG_X86_XADD=y
CONFIG_X86_BSWAP=y
CONFIG_X86_POPAD_OK=y
CONFIG_RWSEM_XCHGADD_ALGORITHM=y
CONFIG_X86_L1_CACHE_SHIFT=5
CONFIG_X86_TSC=y
CONFIG_X86_GOOD_APIC=y
CONFIG_X86_USE_PPRO_CHECKSUM=y
CONFIG_X86_PPRO_FENCE=y
CONFIG_SMP=y
CONFIG_PREEMPT=y
CONFIG_NR_CPUS=2
CONFIG_X86_MCE=y
CONFIG_X86_MCE_NONFATAL=y
CONFIG_X86_MCE_P4THERMAL=y
CONFIG_MICROCODE=y
CONFIG_X86_MSR=y
CONFIG_X86_CPUID=y
CONFIG_HIGHMEM4G=y
CONFIG_HIGHMEM=y
CONFIG_MTRR=y
CONFIG_HAVE_DEC_LOCK=y
CONFIG_ACPI=y
CONFIG_ACPI_BOOT=y
CONFIG_ACPI_AC=y
CONFIG_ACPI_BATTERY=y
CONFIG_ACPI_BUTTON=y
CONFIG_ACPI_FAN=y
CONFIG_ACPI_PROCESSOR=y
CONFIG_ACPI_THERMAL=y
CONFIG_ACPI_BOOT=y
CONFIG_ACPI_BUS=y
CONFIG_ACPI_INTERPRETER=y
CONFIG_ACPI_EC=y
CONFIG_ACPI_POWER=y
CONFIG_ACPI_PCI=y
CONFIG_ACPI_SYSTEM=y
CONFIG_PM=y
CONFIG_X86_IO_APIC=y
CONFIG_X86_LOCAL_APIC=y
CONFIG_PCI=y
CONFIG_PCI_GOANY=y
CONFIG_PCI_BIOS=y
CONFIG_PCI_DIRECT=y
CONFIG_PCI_NAMES=y
CONFIG_ISA=y
CONFIG_KCORE_ELF=y
CONFIG_BINFMT_ELF=y
CONFIG_PARPORT=m
CONFIG_PARPORT_PC=m
CONFIG_PARPORT_PC_CML1=m
CONFIG_PARPORT_PC_FIFO=y
CONFIG_PARPORT_PC_SUPERIO=y
CONFIG_PARPORT_1284=y
CONFIG_PNP=m
CONFIG_BLK_DEV_FD=y
CONFIG_BLK_DEV_LOOP=y
CONFIG_BLK_DEV_NBD=y
CONFIG_BLK_DEV_RAM=y
CONFIG_BLK_DEV_RAM_SIZE=4096
CONFIG_BLK_DEV_INITRD=y
CONFIG_LBD=y
CONFIG_IDE=y
CONFIG_BLK_DEV_IDE=y
CONFIG_BLK_DEV_IDEDISK=y
CONFIG_IDEDISK_MULTI_MODE=y
CONFIG_BLK_DEV_IDECD=y
CONFIG_BLK_DEV_IDESCSI=m
CONFIG_IDE_TASK_IOCTL=y
CONFIG_BLK_DEV_IDEPCI=y
CONFIG_BLK_DEV_GENERIC=y
CONFIG_IDEPCI_SHARE_IRQ=y
CONFIG_BLK_DEV_IDEDMA_PCI=y
CONFIG_IDEDMA_PCI_AUTO=y
CONFIG_BLK_DEV_IDEDMA=y
CONFIG_IDEDMA_PCI_WIP=y
CONFIG_BLK_DEV_ADMA=y
CONFIG_BLK_DEV_PIIX=y
CONFIG_IDEDMA_AUTO=y
CONFIG_BLK_DEV_IDE_MODES=y
CONFIG_SCSI=y
CONFIG_BLK_DEV_SD=y
CONFIG_SD_EXTRA_DEVS=40
CONFIG_CHR_DEV_ST=m
CONFIG_BLK_DEV_SR=m
CONFIG_BLK_DEV_SR_VENDOR=y
CONFIG_SR_EXTRA_DEVS=2
CONFIG_CHR_DEV_SG=m
CONFIG_SCSI_REPORT_LUNS=y
CONFIG_SCSI_CONSTANTS=y
CONFIG_SCSI_AIC7XXX=y
CONFIG_AIC7XXX_CMDS_PER_DEVICE=253
CONFIG_AIC7XXX_RESET_DELAY_MS=15000
CONFIG_MD=y
CONFIG_BLK_DEV_MD=y
CONFIG_MD_LINEAR=y
CONFIG_MD_RAID0=y
CONFIG_MD_RAID1=y
CONFIG_MD_RAID5=y
CONFIG_PACKET=y
CONFIG_PACKET_MMAP=y
CONFIG_NETFILTER=y
CONFIG_FILTER=y
CONFIG_UNIX=y
CONFIG_INET=y
CONFIG_IP_MULTICAST=y
CONFIG_INET_ECN=y
CONFIG_SYN_COOKIES=y
CONFIG_IP_NF_CONNTRACK=m
CONFIG_IP_NF_FTP=m
CONFIG_IP_NF_IRC=m
CONFIG_IP_NF_QUEUE=m
CONFIG_IP_NF_IPTABLES=m
CONFIG_IP_NF_MATCH_LIMIT=m
CONFIG_IP_NF_MATCH_MAC=m
CONFIG_IP_NF_MATCH_PKTTYPE=m
CONFIG_IP_NF_MATCH_MARK=m
CONFIG_IP_NF_MATCH_MULTIPORT=m
CONFIG_IP_NF_MATCH_TOS=m
CONFIG_IP_NF_MATCH_ECN=m
CONFIG_IP_NF_MATCH_DSCP=m
CONFIG_IP_NF_MATCH_AH_ESP=m
CONFIG_IP_NF_MATCH_LENGTH=m
CONFIG_IP_NF_MATCH_TTL=m
CONFIG_IP_NF_MATCH_TCPMSS=m
CONFIG_IP_NF_MATCH_HELPER=m
CONFIG_IP_NF_MATCH_STATE=m
CONFIG_IP_NF_MATCH_CONNTRACK=m
CONFIG_IP_NF_MATCH_UNCLEAN=m
CONFIG_IP_NF_MATCH_OWNER=m
CONFIG_IP_NF_FILTER=m
CONFIG_IP_NF_TARGET_REJECT=m
CONFIG_IP_NF_TARGET_MIRROR=m
CONFIG_IP_NF_MANGLE=m
CONFIG_IP_NF_TARGET_TOS=m
CONFIG_IP_NF_TARGET_ECN=m
CONFIG_IP_NF_TARGET_DSCP=m
CONFIG_IP_NF_TARGET_MARK=m
CONFIG_IP_NF_TARGET_LOG=m
CONFIG_IP_NF_TARGET_ULOG=m
CONFIG_IP_NF_TARGET_TCPMSS=m
CONFIG_IP_NF_ARPTABLES=m
CONFIG_IP_NF_ARPFILTER=m
CONFIG_IPV6_SCTP__=y
CONFIG_NETDEVICES=y
CONFIG_DUMMY=m
CONFIG_TUN=y
CONFIG_NET_ETHERNET=y
CONFIG_NET_VENDOR_3COM=y
CONFIG_VORTEX=y
CONFIG_NET_PCI=y
CONFIG_E100=y
CONFIG_PPP=m
CONFIG_PPP_MULTILINK=y
CONFIG_PPP_FILTER=y
CONFIG_PPP_ASYNC=m
CONFIG_PPP_DEFLATE=m
CONFIG_PPP_BSDCOMP=m
CONFIG_PPPOE=m
CONFIG_INPUT=y
CONFIG_INPUT_MOUSEDEV=y
CONFIG_INPUT_MOUSEDEV_PSAUX=y
CONFIG_INPUT_MOUSEDEV_SCREEN_X=1024
CONFIG_INPUT_MOUSEDEV_SCREEN_Y=768
CONFIG_SOUND_GAMEPORT=y
CONFIG_SERIO=y
CONFIG_SERIO_I8042=y
CONFIG_INPUT_KEYBOARD=y
CONFIG_KEYBOARD_ATKBD=y
CONFIG_INPUT_MOUSE=y
CONFIG_MOUSE_PS2=y
CONFIG_INPUT_MISC=y
CONFIG_INPUT_PCSPKR=y
CONFIG_VT=y
CONFIG_VT_CONSOLE=y
CONFIG_HW_CONSOLE=y
CONFIG_SERIAL_8250=y
CONFIG_SERIAL_8250_CONSOLE=y
CONFIG_SERIAL_CORE=y
CONFIG_SERIAL_CORE_CONSOLE=y
CONFIG_UNIX98_PTYS=y
CONFIG_UNIX98_PTY_COUNT=256
CONFIG_PRINTER=m
CONFIG_PPDEV=m
CONFIG_RTC=y
CONFIG_RAW_DRIVER=y
CONFIG_REISERFS_FS=y
CONFIG_EXT3_FS=y
CONFIG_JBD=y
CONFIG_FAT_FS=m
CONFIG_MSDOS_FS=m
CONFIG_VFAT_FS=m
CONFIG_TMPFS=y
CONFIG_RAMFS=y
CONFIG_ISO9660_FS=y
CONFIG_JOLIET=y
CONFIG_JFS_FS=y
CONFIG_MINIX_FS=y
CONFIG_NTFS_FS=m
CONFIG_NTFS_RW=y
CONFIG_PROC_FS=y
CONFIG_DEVPTS_FS=y
CONFIG_EXT2_FS=y
CONFIG_NFS_FS=y
CONFIG_NFS_V3=y
CONFIG_NFSD=y
CONFIG_NFSD_V3=y
CONFIG_NFSD_TCP=y
CONFIG_SUNRPC=y
CONFIG_LOCKD=y
CONFIG_LOCKD_V4=y
CONFIG_EXPORTFS=y
CONFIG_SMB_FS=m
CONFIG_MSDOS_PARTITION=y
CONFIG_SMB_NLS=y
CONFIG_NLS=y
CONFIG_NLS_DEFAULT="iso8859-1"
CONFIG_NLS_CODEPAGE_437=m
CONFIG_NLS_ISO8859_1=m
CONFIG_VGA_CONSOLE=y
CONFIG_DEBUG_KERNEL=y
CONFIG_MAGIC_SYSRQ=y
CONFIG_KALLSYMS=y
CONFIG_X86_EXTRA_IRQS=y
CONFIG_X86_FIND_SMP_CONFIG=y
CONFIG_X86_MPPARSE=y
CONFIG_SECURITY_CAPABILITIES=y
CONFIG_CRC32=y
CONFIG_ZLIB_INFLATE=m
CONFIG_ZLIB_DEFLATE=m
CONFIG_X86_SMP=y
CONFIG_X86_HT=y
CONFIG_X86_BIOS_REBOOT=y

 

-- 
/==============================\
| David Mansfield              |
| lkml@dm.cobite.com           |
\==============================/

Re: [BUG] raw over raid5: BUG at drivers/block/ll_rw_blk.c:1967

[Andrew Morton]

David Mansfield wrote:
> 
> Hi everyone,
> 
> I haven't been able to run raw over raid5 since 2.5.30 or so, but every
> time I'm about to report it, a new kernel comes out and the problem
> changes completely :-( Now I'm finally going to start getting out the info
> it the hopes someone can fix it.  The oops was triggered by attempting to
> read from /dev/raw/raw1 (bound to /dev/md0) using dd.  System info
> follows oops:
> 
> ------------[ cut here ]------------
> kernel BUG at drivers/block/ll_rw_blk.c:1967!

I don't think you told us the kernel version?

There have been recent fixes wrt sizing of the BIOs which the
direct-io layer sends down.  So please make sure that you're
testing Linus's current -bk, or 2.5.42 plus

http://www.zip.com.au/~akpm/linux/patches/2.5/2.5.42/2.5.42-mm2/broken-out/dio-bio-add-fix-1.patch

Either that, or raid5 is bust ;)

Re: [BUG] raw over raid5: BUG at drivers/block/ll_rw_blk.c:1967

[David Mansfield]

On Mon, 14 Oct 2002, Andrew Morton wrote:

> David Mansfield wrote:
> > 
> > Hi everyone,
> > 
> > I haven't been able to run raw over raid5 since 2.5.30 or so, but every
> > time I'm about to report it, a new kernel comes out and the problem
> > changes completely :-( Now I'm finally going to start getting out the info
> > it the hopes someone can fix it.  The oops was triggered by attempting to
> > read from /dev/raw/raw1 (bound to /dev/md0) using dd.  System info
> > follows oops:
> > 
> > ------------[ cut here ]------------
> > kernel BUG at drivers/block/ll_rw_blk.c:1967!
> 
> I don't think you told us the kernel version?

Arrgh.  It was 2.5.42 vanilla.

> There have been recent fixes wrt sizing of the BIOs which the
> direct-io layer sends down.  So please make sure that you're
> testing Linus's current -bk, or 2.5.42 plus
> 
> http://www.zip.com.au/~akpm/linux/patches/2.5/2.5.42/2.5.42-mm2/broken-out/dio-bio-add-fix-1.patch

I've applied this patch and retested.  I was able to run dd once this 
time, hit ctrl-c, all ok.  But I hit up-arrow and re-did it and it gave 
the same oops.  I'm going to try with the full (latest) mm patch.

> Either that, or raid5 is bust ;)
> 

I guess so.

David

-- 
/==============================\
| David Mansfield              |
| lkml@dm.cobite.com           |
\==============================/

Re: [BUG] raw over raid5: BUG at drivers/block/ll_rw_blk.c:1967

[David Mansfield]

On Mon, 14 Oct 2002, Andrew Morton wrote:

> David Mansfield wrote:
> > ------------[ cut here ]------------
> > kernel BUG at drivers/block/ll_rw_blk.c:1967!
> 
> I don't think you told us the kernel version?
> 
> There have been recent fixes wrt sizing of the BIOs which the
> direct-io layer sends down.  So please make sure that you're
> testing Linus's current -bk, or 2.5.42 plus
> 

I've tried to test 2.5.42-mm2 but it doesn't compile.  There were (I 
assume 'sard') changes made to remove the DK_MAX_MAJOR etc. but there is 
to patch to drivers/md/md.c so it fails to compile:

# find -name \*.c | xargs grep DK_MAX_MAJOR
./drivers/md/md.c:static unsigned int sync_io[DK_MAX_MAJOR][DK_MAX_DISK];
./drivers/md/md.c:      if ((index >= DK_MAX_DISK) || (major >= 
DK_MAX_MAJOR))
./drivers/md/md.c:              if ((idx >= DK_MAX_DISK) || (major >= 
DK_MAX_MAJOR))

David

-- 
/==============================\
| David Mansfield              |
| david@cobite.com             |
\==============================/

Re: [BUG] raw over raid5: BUG at drivers/block/ll_rw_blk.c:1967

[Neil Brown]

On Monday October 14, lkml@dm.cobite.com wrote:
> 
> Hi everyone,
> 
> I haven't been able to run raw over raid5 since 2.5.30 or so, but every
> time I'm about to report it, a new kernel comes out and the problem
> changes completely :-( Now I'm finally going to start getting out the info
> it the hopes someone can fix it.  The oops was triggered by attempting to 
> read from /dev/raw/raw1 (bound to /dev/md0) using dd.  System info 
> follows oops:
> 
> ------------[ cut here ]------------
> kernel BUG at drivers/block/ll_rw_blk.c:1967!
> invalid operand: 0000
>  

You are not alone in reporting this BUG...

I blame the Scsi/bio  layer.
Jens Axboe blames raid5.
:-)

Hopefully we will find it one place or the other. 
The other report didn't use raw to get the bug, but if using raw makes
it reproducable, I will try that and see if I can reproduce it easily.

NeilBrown

Re: [BUG] raw over raid5: BUG at drivers/block/ll_rw_blk.c:1967

[Neil Brown]

On Tuesday October 15, neilb@cse.unsw.edu.au wrote:
> On Monday October 14, lkml@dm.cobite.com wrote:
> > 
> > Hi everyone,
> > 
> > I haven't been able to run raw over raid5 since 2.5.30 or so, but every
> > time I'm about to report it, a new kernel comes out and the problem
> > changes completely :-( Now I'm finally going to start getting out the info
> > it the hopes someone can fix it.  The oops was triggered by attempting to 
> > read from /dev/raw/raw1 (bound to /dev/md0) using dd.  System info 
> > follows oops:
> > 
> > ------------[ cut here ]------------
> > kernel BUG at drivers/block/ll_rw_blk.c:1967!
> > invalid operand: 0000
> >  
> 
> You are not alone in reporting this BUG...
> 
> I blame the Scsi/bio  layer.
> Jens Axboe blames raid5.
> :-)
> 

Just for the record, Jens was right. :-)
Here is the patch which will be winging it's way to Linus shortly.

NeilBrown
-----------------------------
Fix bug in raid5

When analysing a stripe in handle_stripe we set bits
 R5_Wantread or R5_Wantwrite
to indicate if a read or write is needed.  We don't actually schedule the
IO immediately as this is done under a spinlock (sh->lock) and 
generic_make_request can block.  Instead we check these bits after
the lock has been lifted and then schedule the IO.

But once the lock has been lifted we aren't safe against multiple
access, and it is possible that the IO will be scheduled never, or twice.

So, we use test_and_clear to check and potentially schedule the IO.

This wasn't a problem in 2.4 because the equivalent information was
stored on the stack instead of in the stripe.

We also make sure bi_io_vec[0] has correct values as a previous
call to generic_make_request may have changed them.

 ----------- Diffstat output ------------
 ./drivers/md/raid5.c |   92 +++++++++++++++++++++++++++------------------------
 1 file changed, 51 insertions(+), 43 deletions(-)

--- ./drivers/md/raid5.c	2002/11/05 23:31:25	1.1
+++ ./drivers/md/raid5.c	2002/11/05 23:39:15	1.2
@@ -851,8 +851,6 @@ static void handle_stripe(struct stripe_
 	for (i=disks; i--; ) {
 		mdk_rdev_t *rdev;
 		dev = &sh->dev[i];
-		clear_bit(R5_Wantread, &dev->flags);
-		clear_bit(R5_Wantwrite, &dev->flags);
 		clear_bit(R5_Insync, &dev->flags);
 		clear_bit(R5_Syncio, &dev->flags);
 
@@ -1160,48 +1158,56 @@ static void handle_stripe(struct stripe_
 		bi->bi_size = 0;
 		bi->bi_end_io(bi, bytes, 0);
 	}
-	for (i=disks; i-- ;) 
-		if (sh->dev[i].flags & ((1<<R5_Wantwrite)|(1<<R5_Wantread))) {
-			struct bio *bi = &sh->dev[i].req;
-			mdk_rdev_t *rdev ;
-
-			bi->bi_rw = 0;
-			if (test_bit(R5_Wantread, &sh->dev[i].flags))
-				bi->bi_end_io = raid5_end_read_request;
-			else {
-				bi->bi_end_io = raid5_end_write_request;
-				bi->bi_rw = 1;
-			}
-
-			spin_lock_irq(&conf->device_lock);
-			rdev = conf->disks[i].rdev;
-			if (rdev && rdev->faulty)
-				rdev = NULL;
-			if (rdev)
-				atomic_inc(&rdev->nr_pending);
-			spin_unlock_irq(&conf->device_lock);
-
-			if (rdev) {
-				if (test_bit(R5_Syncio, &sh->dev[i].flags))
-					md_sync_acct(rdev, STRIPE_SECTORS);
-
-				bi->bi_bdev = rdev->bdev;
-				PRINTK("for %llu schedule op %ld on disc %d\n", (unsigned long long)sh->sector, bi->bi_rw, i);
-				atomic_inc(&sh->count);
-				bi->bi_sector = sh->sector;
-				bi->bi_flags = 1 << BIO_UPTODATE;
-				bi->bi_vcnt = 1;	
-				bi->bi_idx = 0;
-				bi->bi_io_vec = &sh->dev[i].vec;
-				bi->bi_size = STRIPE_SIZE;
-				bi->bi_next = NULL;
-				generic_make_request(bi);
-			} else {
-				PRINTK("skip op %ld on disc %d for sector %llu\n", bi->bi_rw, i, (unsigned long long)sh->sector);
-				clear_bit(R5_LOCKED, &dev->flags);
-				set_bit(STRIPE_HANDLE, &sh->state);
-			}
+	for (i=disks; i-- ;) {
+		int rw;
+		struct bio *bi;
+		mdk_rdev_t *rdev;
+		if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags))
+			rw = 1;
+		else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
+			rw = 0;
+		else
+			continue;
+ 
+		bi = &sh->dev[i].req;
+ 
+		bi->bi_rw = rw;
+		if (rw)
+			bi->bi_end_io = raid5_end_write_request;
+		else
+			bi->bi_end_io = raid5_end_read_request;
+ 
+		spin_lock_irq(&conf->device_lock);
+		rdev = conf->disks[i].rdev;
+		if (rdev && rdev->faulty)
+			rdev = NULL;
+		if (rdev)
+			atomic_inc(&rdev->nr_pending);
+		spin_unlock_irq(&conf->device_lock);
+ 
+		if (rdev) {
+			if (test_bit(R5_Syncio, &sh->dev[i].flags))
+				md_sync_acct(rdev, STRIPE_SECTORS);
+
+			bi->bi_bdev = rdev->bdev;
+			PRINTK("for %llu schedule op %ld on disc %d\n", (unsigned long long)sh->sector, bi->bi_rw, i);
+			atomic_inc(&sh->count);
+			bi->bi_sector = sh->sector;
+			bi->bi_flags = 1 << BIO_UPTODATE;
+			bi->bi_vcnt = 1;	
+			bi->bi_idx = 0;
+			bi->bi_io_vec = &sh->dev[i].vec;
+			bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
+			bi->bi_io_vec[0].bv_offset = 0;
+			bi->bi_size = STRIPE_SIZE;
+			bi->bi_next = NULL;
+			generic_make_request(bi);
+		} else {
+			PRINTK("skip op %ld on disc %d for sector %llu\n", bi->bi_rw, i, (unsigned long long)sh->sector);
+			clear_bit(R5_LOCKED, &dev->flags);
+			set_bit(STRIPE_HANDLE, &sh->state);
 		}
+	}
 }
 
 static inline void raid5_activate_delayed(raid5_conf_t *conf)