[PATCH 0/1] NFSv4.1: pNFS has no business touching the slot table waitq...

[PATCH 0/1] NFSv4.1: pNFS has no business touching the slot table waitq...

[andros]

From: Andy Adamson <andros@netapp.com>

Hi Trond:

How about this?

I tested this patch against the rpc_wake_up version - no difference in run
time for DS disconnect.

I have not yet tested DS invalid layout, but since
the invalid layout does not trigger a deviceid invalidation, each call using
the layout will go through filelayout_async_handle_error, free the slot, and
wake up the next task on the ds session fore channel slot table waitq, and
the filelayout_read/write_prepare routines do not reset to the MDS.

-->Andy

Andy Adamson (1):
  NFSv4.1	do not wake up all tasks on data server reset to MDS

 fs/nfs/internal.h       |    1 +
 fs/nfs/nfs4filelayout.c |    5 ++---
 fs/nfs/nfs4proc.c       |   18 +++++++++++++-----
 3 files changed, 16 insertions(+), 8 deletions(-)

-- 
1.7.7.6

[PATCH 1/1] NFSv4.1 do not wake up all tasks on data server reset to MDS

[andros]

From: Andy Adamson <andros@netapp.com>

There is no advantage to waking up all rpc tasks on each data server disconnect
or invalid layout error.

For DS disconnect errors where the data server is reseting all I/O to the MDS,
tasks wakened from the fore channel slot table waitq are immediately reset
to use the MDS session, and do not request a data server session slot, and
so do not call nfs41_session_free_slot, and do not wake up the next task on
the data server session fore channel slot table wait queue.

Replace the call to rpc_wake_up which wakes up all tasks with a call to
rpc_wake_up_first in the filelayout I/O prepare routines for the case where
I/O is redirected to the MDS.

Share code with nfs4_check_drain_fc_complete.

Reported-by: Trond Myklebust <trond.myklebust@netapp.com>
Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfs/internal.h       |    1 +
 fs/nfs/nfs4filelayout.c |    5 ++---
 fs/nfs/nfs4proc.c       |   18 +++++++++++++-----
 3 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 59b133c..0b48ce8 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -490,6 +490,7 @@ extern int nfs40_walk_client_list(struct nfs_client *clp,
 extern int nfs41_walk_client_list(struct nfs_client *clp,
 				struct nfs_client **result,
 				struct rpc_cred *cred);
+extern bool nfs4_wake_up_first(struct nfs4_session *ses);
 
 /*
  * Determine the device name as a string
diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
index 52d8472..d52c7d0 100644
--- a/fs/nfs/nfs4filelayout.c
+++ b/fs/nfs/nfs4filelayout.c
@@ -131,7 +131,6 @@ static int filelayout_async_handle_error(struct rpc_task *task,
 	struct nfs_server *mds_server = NFS_SERVER(inode);
 	struct nfs4_deviceid_node *devid = FILELAYOUT_DEVID_NODE(lseg);
 	struct nfs_client *mds_client = mds_server->nfs_client;
-	struct nfs4_slot_table *tbl = &clp->cl_session->fc_slot_table;
 
 	if (task->tk_status >= 0)
 		return 0;
@@ -191,7 +190,6 @@ static int filelayout_async_handle_error(struct rpc_task *task,
 		 * layout is destroyed and a new valid layout is obtained.
 		 */
 		pnfs_destroy_layout(NFS_I(inode));
-		rpc_wake_up(&tbl->slot_tbl_waitq);
 		goto reset;
 	/* RPC connection errors */
 	case -ECONNREFUSED:
@@ -206,7 +204,6 @@ static int filelayout_async_handle_error(struct rpc_task *task,
 		nfs4_mark_deviceid_unavailable(devid);
 		clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(inode)->flags);
 		_pnfs_return_layout(inode);
-		rpc_wake_up(&tbl->slot_tbl_waitq);
 		nfs4_ds_disconnect(clp);
 		/* fall through */
 	default:
@@ -294,6 +291,7 @@ static void filelayout_read_prepare(struct rpc_task *task, void *data)
 	if (filelayout_reset_to_mds(rdata->header->lseg)) {
 		dprintk("%s task %u reset io to MDS\n", __func__, task->tk_pid);
 		filelayout_reset_read(rdata);
+		nfs4_wake_up_first(rdata->ds_clp->cl_session);
 		rpc_exit(task, 0);
 		return;
 	}
@@ -396,6 +394,7 @@ static void filelayout_write_prepare(struct rpc_task *task, void *data)
 	if (filelayout_reset_to_mds(wdata->header->lseg)) {
 		dprintk("%s task %u reset io to MDS\n", __func__, task->tk_pid);
 		filelayout_reset_write(wdata);
+		nfs4_wake_up_first(wdata->ds_clp->cl_session);
 		rpc_exit(task, 0);
 		return;
 	}
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 68b21d8..f0c0a12 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -438,17 +438,25 @@ bool nfs4_set_task_privileged(struct rpc_task *task, void *dummy)
 	return true;
 }
 
-/*
- * Signal state manager thread if session fore channel is drained
- */
-static void nfs4_check_drain_fc_complete(struct nfs4_session *ses)
+bool nfs4_wake_up_first(struct nfs4_session *ses)
 {
 	if (!test_bit(NFS4_SESSION_DRAINING, &ses->session_state)) {
 		rpc_wake_up_first(&ses->fc_slot_table.slot_tbl_waitq,
 				nfs4_set_task_privileged, NULL);
-		return;
+		return true;
 	}
+	return false;
+}
+EXPORT_SYMBOL_GPL(nfs4_wake_up_first);
 
+/*
+ * Signal state manager thread if session fore channel is drained
+ */
+static void nfs4_check_drain_fc_complete(struct nfs4_session *ses)
+{
+
+	if (nfs4_wake_up_first(ses))
+		return;
 	if (ses->fc_slot_table.highest_used_slotid != NFS4_NO_SLOT)
 		return;
 
-- 
1.7.7.6

Re: [PATCH 1/1] NFSv4.1 do not wake up all tasks on data server reset to MDS

[Myklebust, Trond]

T24gVHVlLCAyMDEyLTEwLTA5IGF0IDE3OjIwIC0wNDAwLCBhbmRyb3NAbmV0YXBwLmNvbSB3cm90
ZToNCj4gRnJvbTogQW5keSBBZGFtc29uIDxhbmRyb3NAbmV0YXBwLmNvbT4NCj4gDQo+IFRoZXJl
IGlzIG5vIGFkdmFudGFnZSB0byB3YWtpbmcgdXAgYWxsIHJwYyB0YXNrcyBvbiBlYWNoIGRhdGEg
c2VydmVyIGRpc2Nvbm5lY3QNCj4gb3IgaW52YWxpZCBsYXlvdXQgZXJyb3IuDQo+IA0KPiBGb3Ig
RFMgZGlzY29ubmVjdCBlcnJvcnMgd2hlcmUgdGhlIGRhdGEgc2VydmVyIGlzIHJlc2V0aW5nIGFs
bCBJL08gdG8gdGhlIE1EUywNCj4gdGFza3Mgd2FrZW5lZCBmcm9tIHRoZSBmb3JlIGNoYW5uZWwg
c2xvdCB0YWJsZSB3YWl0cSBhcmUgaW1tZWRpYXRlbHkgcmVzZXQNCj4gdG8gdXNlIHRoZSBNRFMg
c2Vzc2lvbiwgYW5kIGRvIG5vdCByZXF1ZXN0IGEgZGF0YSBzZXJ2ZXIgc2Vzc2lvbiBzbG90LCBh
bmQNCj4gc28gZG8gbm90IGNhbGwgbmZzNDFfc2Vzc2lvbl9mcmVlX3Nsb3QsIGFuZCBkbyBub3Qg
d2FrZSB1cCB0aGUgbmV4dCB0YXNrIG9uDQo+IHRoZSBkYXRhIHNlcnZlciBzZXNzaW9uIGZvcmUg
Y2hhbm5lbCBzbG90IHRhYmxlIHdhaXQgcXVldWUuDQo+IA0KPiBSZXBsYWNlIHRoZSBjYWxsIHRv
IHJwY193YWtlX3VwIHdoaWNoIHdha2VzIHVwIGFsbCB0YXNrcyB3aXRoIGEgY2FsbCB0bw0KPiBy
cGNfd2FrZV91cF9maXJzdCBpbiB0aGUgZmlsZWxheW91dCBJL08gcHJlcGFyZSByb3V0aW5lcyBm
b3IgdGhlIGNhc2Ugd2hlcmUNCj4gSS9PIGlzIHJlZGlyZWN0ZWQgdG8gdGhlIE1EUy4NCj4gDQo+
IFNoYXJlIGNvZGUgd2l0aCBuZnM0X2NoZWNrX2RyYWluX2ZjX2NvbXBsZXRlLg0KDQpUaGlzIHdp
bGwgc3RpbGwgaGF2ZSBmYWxzZSBwb3NpdGl2ZSB3YWtldXBzLCB0aG91Z2gsIHNpbmNlIHRoZXJl
IGlzIG5vDQpjb25uZWN0aW9uIHRvIG51bWJlciBvZiBzbG90cyB0aGF0IGFyZSBhdmFpbGFibGUu
DQoNCkhvdyBhYm91dCByYXRoZXIganVzdCBsZXR0aW5nIHRoZSBzZXNzaW9uIGNvZGUgZHJpdmUg
dGhlIHdha2V1cHM/IEknbQ0KdGhpbmtpbmcgc29tZXRoaW5nIGxpa2UgdGhlIGZvbGxvd2luZyBw
YXRjaDoNCg0KODwtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0tLQ0KZGlmZiAtLWdpdCBhL2ZzL25mcy9uZnM0X2ZzLmggYi9mcy9uZnMv
bmZzNF9mcy5oDQppbmRleCBhNTI1ZmRlLi5kZjI0NzRiIDEwMDY0NA0KLS0tIGEvZnMvbmZzL25m
czRfZnMuaA0KKysrIGIvZnMvbmZzL25mczRfZnMuaA0KQEAgLTI0OCw2ICsyNDgsNyBAQCBleHRl
cm4gaW50IG5mczRfc2V0dXBfc2VxdWVuY2UoY29uc3Qgc3RydWN0IG5mc19zZXJ2ZXIgKnNlcnZl
ciwNCiBleHRlcm4gaW50IG5mczQxX3NldHVwX3NlcXVlbmNlKHN0cnVjdCBuZnM0X3Nlc3Npb24g
KnNlc3Npb24sDQogCQlzdHJ1Y3QgbmZzNF9zZXF1ZW5jZV9hcmdzICphcmdzLCBzdHJ1Y3QgbmZz
NF9zZXF1ZW5jZV9yZXMgKnJlcywNCiAJCXN0cnVjdCBycGNfdGFzayAqdGFzayk7DQorZXh0ZXJu
IHZvaWQgbmZzNDFfc2VxdWVuY2VfZnJlZV9zbG90KHN0cnVjdCBuZnM0X3NlcXVlbmNlX3JlcyAq
cmVzKTsNCiBleHRlcm4gdm9pZCBuZnM0X2Rlc3Ryb3lfc2Vzc2lvbihzdHJ1Y3QgbmZzNF9zZXNz
aW9uICpzZXNzaW9uKTsNCiBleHRlcm4gc3RydWN0IG5mczRfc2Vzc2lvbiAqbmZzNF9hbGxvY19z
ZXNzaW9uKHN0cnVjdCBuZnNfY2xpZW50ICpjbHApOw0KIGV4dGVybiBpbnQgbmZzNF9wcm9jX2Ny
ZWF0ZV9zZXNzaW9uKHN0cnVjdCBuZnNfY2xpZW50ICosIHN0cnVjdCBycGNfY3JlZCAqKTsNCmRp
ZmYgLS1naXQgYS9mcy9uZnMvbmZzNGZpbGVsYXlvdXQuYyBiL2ZzL25mcy9uZnM0ZmlsZWxheW91
dC5jDQppbmRleCA1MmQ4NDcyLi44MGFhMTlhIDEwMDY0NA0KLS0tIGEvZnMvbmZzL25mczRmaWxl
bGF5b3V0LmMNCisrKyBiL2ZzL25mcy9uZnM0ZmlsZWxheW91dC5jDQpAQCAtMjkxLDE5ICsyOTEs
MjAgQEAgc3RhdGljIHZvaWQgZmlsZWxheW91dF9yZWFkX3ByZXBhcmUoc3RydWN0IHJwY190YXNr
ICp0YXNrLCB2b2lkICpkYXRhKQ0KIHsNCiAJc3RydWN0IG5mc19yZWFkX2RhdGEgKnJkYXRhID0g
ZGF0YTsNCiANCisJaWYgKG5mczQxX3NldHVwX3NlcXVlbmNlKHJkYXRhLT5kc19jbHAtPmNsX3Nl
c3Npb24sDQorCQkJCSZyZGF0YS0+YXJncy5zZXFfYXJncywgJnJkYXRhLT5yZXMuc2VxX3JlcywN
CisJCQkJdGFzaykpDQorCQlyZXR1cm47DQorDQogCWlmIChmaWxlbGF5b3V0X3Jlc2V0X3RvX21k
cyhyZGF0YS0+aGVhZGVyLT5sc2VnKSkgew0KIAkJZHByaW50aygiJXMgdGFzayAldSByZXNldCBp
byB0byBNRFNcbiIsIF9fZnVuY19fLCB0YXNrLT50a19waWQpOw0KIAkJZmlsZWxheW91dF9yZXNl
dF9yZWFkKHJkYXRhKTsNCisJCW5mczQxX3NlcXVlbmNlX2ZyZWVfc2xvdCgmcmRhdGEtPnJlcy5z
ZXFfcmVzKTsNCiAJCXJwY19leGl0KHRhc2ssIDApOw0KIAkJcmV0dXJuOw0KIAl9DQogCXJkYXRh
LT5yZWFkX2RvbmVfY2IgPSBmaWxlbGF5b3V0X3JlYWRfZG9uZV9jYjsNCiANCi0JaWYgKG5mczQx
X3NldHVwX3NlcXVlbmNlKHJkYXRhLT5kc19jbHAtPmNsX3Nlc3Npb24sDQotCQkJCSZyZGF0YS0+
YXJncy5zZXFfYXJncywgJnJkYXRhLT5yZXMuc2VxX3JlcywNCi0JCQkJdGFzaykpDQotCQlyZXR1
cm47DQotDQogCXJwY19jYWxsX3N0YXJ0KHRhc2spOw0KIH0NCiANCkBAIC0zOTMsMTcgKzM5NCwx
OCBAQCBzdGF0aWMgdm9pZCBmaWxlbGF5b3V0X3dyaXRlX3ByZXBhcmUoc3RydWN0IHJwY190YXNr
ICp0YXNrLCB2b2lkICpkYXRhKQ0KIHsNCiAJc3RydWN0IG5mc193cml0ZV9kYXRhICp3ZGF0YSA9
IGRhdGE7DQogDQorCWlmIChuZnM0MV9zZXR1cF9zZXF1ZW5jZSh3ZGF0YS0+ZHNfY2xwLT5jbF9z
ZXNzaW9uLA0KKwkJCQkmd2RhdGEtPmFyZ3Muc2VxX2FyZ3MsICZ3ZGF0YS0+cmVzLnNlcV9yZXMs
DQorCQkJCXRhc2spKQ0KKwkJcmV0dXJuOw0KKw0KIAlpZiAoZmlsZWxheW91dF9yZXNldF90b19t
ZHMod2RhdGEtPmhlYWRlci0+bHNlZykpIHsNCiAJCWRwcmludGsoIiVzIHRhc2sgJXUgcmVzZXQg
aW8gdG8gTURTXG4iLCBfX2Z1bmNfXywgdGFzay0+dGtfcGlkKTsNCiAJCWZpbGVsYXlvdXRfcmVz
ZXRfd3JpdGUod2RhdGEpOw0KKwkJbmZzNDFfc2VxdWVuY2VfZnJlZV9zbG90KCZyZGF0YS0+cmVz
LnNlcV9yZXMpOw0KIAkJcnBjX2V4aXQodGFzaywgMCk7DQogCQlyZXR1cm47DQogCX0NCi0JaWYg
KG5mczQxX3NldHVwX3NlcXVlbmNlKHdkYXRhLT5kc19jbHAtPmNsX3Nlc3Npb24sDQotCQkJCSZ3
ZGF0YS0+YXJncy5zZXFfYXJncywgJndkYXRhLT5yZXMuc2VxX3JlcywNCi0JCQkJdGFzaykpDQot
CQlyZXR1cm47DQotDQogCXJwY19jYWxsX3N0YXJ0KHRhc2spOw0KIH0NCiANCmRpZmYgLS1naXQg
YS9mcy9uZnMvbmZzNHByb2MuYyBiL2ZzL25mcy9uZnM0cHJvYy5jDQppbmRleCA2OGIyMWQ4Li5k
MTg2YmUyIDEwMDY0NA0KLS0tIGEvZnMvbmZzL25mczRwcm9jLmMNCisrKyBiL2ZzL25mcy9uZnM0
cHJvYy5jDQpAQCAtNDY4LDcgKzQ2OCw3IEBAIHZvaWQgbmZzNF9jaGVja19kcmFpbl9iY19jb21w
bGV0ZShzdHJ1Y3QgbmZzNF9zZXNzaW9uICpzZXMpDQogCWNvbXBsZXRlKCZzZXMtPmJjX3Nsb3Rf
dGFibGUuY29tcGxldGUpOw0KIH0NCiANCi1zdGF0aWMgdm9pZCBuZnM0MV9zZXF1ZW5jZV9mcmVl
X3Nsb3Qoc3RydWN0IG5mczRfc2VxdWVuY2VfcmVzICpyZXMpDQordm9pZCBuZnM0MV9zZXF1ZW5j
ZV9mcmVlX3Nsb3Qoc3RydWN0IG5mczRfc2VxdWVuY2VfcmVzICpyZXMpDQogew0KIAlzdHJ1Y3Qg
bmZzNF9zbG90X3RhYmxlICp0Ymw7DQogDQpAQCAtNDg2LDYgKzQ4Niw3IEBAIHN0YXRpYyB2b2lk
IG5mczQxX3NlcXVlbmNlX2ZyZWVfc2xvdChzdHJ1Y3QgbmZzNF9zZXF1ZW5jZV9yZXMgKnJlcykN
CiAJc3Bpbl91bmxvY2soJnRibC0+c2xvdF90YmxfbG9jayk7DQogCXJlcy0+c3Jfc2xvdCA9IE5V
TEw7DQogfQ0KK0VYUE9SVF9TWU1CT0xfR1BMKG5mczQxX3NlcXVlbmNlX2ZyZWVfc2xvdCk7DQog
DQogc3RhdGljIGludCBuZnM0MV9zZXF1ZW5jZV9kb25lKHN0cnVjdCBycGNfdGFzayAqdGFzaywg
c3RydWN0IG5mczRfc2VxdWVuY2VfcmVzICpyZXMpDQogew0KDQotLSANClRyb25kIE15a2xlYnVz
dA0KTGludXggTkZTIGNsaWVudCBtYWludGFpbmVyDQoNCk5ldEFwcA0KVHJvbmQuTXlrbGVidXN0
QG5ldGFwcC5jb20NCnd3dy5uZXRhcHAuY29tDQo=

Re: [PATCH 1/1] NFSv4.1 do not wake up all tasks on data server reset to MDS

[Adamson, Andy]

On Oct 10, 2012, at 2:06 PM, Myklebust, Trond wrote:

> On Tue, 2012-10-09 at 17:20 -0400, andros@netapp.com wrote:
>> From: Andy Adamson <andros@netapp.com>
>> 
>> There is no advantage to waking up all rpc tasks on each data server disconnect
>> or invalid layout error.
>> 
>> For DS disconnect errors where the data server is reseting all I/O to the MDS,
>> tasks wakened from the fore channel slot table waitq are immediately reset
>> to use the MDS session, and do not request a data server session slot, and
>> so do not call nfs41_session_free_slot, and do not wake up the next task on
>> the data server session fore channel slot table wait queue.
>> 
>> Replace the call to rpc_wake_up which wakes up all tasks with a call to
>> rpc_wake_up_first in the filelayout I/O prepare routines for the case where
>> I/O is redirected to the MDS.
>> 
>> Share code with nfs4_check_drain_fc_complete.
> 
> This will still have false positive wakeups, though, since there is no
> connection to number of slots that are available.

I can't see that number of slots available is an issue. The Deviceid is bad due to a DS connection error.

case 1) DS Role:  All of the tasks are to be redirected to a (working) MDS session on another nfs_client. None of the disconnected DS session slots will be used, no reason to wait for a slot that will not be used.

case 2) MDS DS Role, and MDS is the DS's MDS:  No chance of recovery, as there is no working MDS to redirect to. Wait for a slot, don't wait for a slot, no difference.

case 3) MDS DS Role, and MDS is not the DS's MDS: Here is where this patch fails, and come to think of it, rpc_wake_up is the way to go. The disconnected session slot_table_waitq can contain both MDS and DS traffic. The MDS traffic is hard mounted and will fail (disconnected). So, having the DS tasks which can be redirected to a (working) MDS session wait behind failed hard mounted MDS tasks will not work as the hard mounted MDS tasks will retry forever starving the DS tasks, and we could end up with a bunch of DS tasks still on the disconnected session wait queue with all MDS tasks taking the slots.

My patch which calls rpc_wake_up_first in filelayout_read/write_prepare can also result in DS tasks being starved behind MDS hard mounted tasks as failed DS tasks give up their slot via the session code, and so MDS tasks could use all the slots and DS recovery will fail. 

This is why the rpc_wake_up call in filelayout_async_handle_error for the DS disconnect case is correct. It separates the DS from the MDS traffic guaranteeing all DS tasks will be redirected. The "churn" makes no difference in case 1 as all tasks will be redirected or in case 2 as no tasks will succeed.

So I vote for keeping the rpc_wake_up in the DS disconnect case.

-->Andy

> 
> How about rather just letting the session code drive the wakeups? I'm
> thinking something like the following patch:
> 
> 8<--------------------------------------------------------------
> diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
> index a525fde..df2474b 100644
> --- a/fs/nfs/nfs4_fs.h
> +++ b/fs/nfs/nfs4_fs.h
> @@ -248,6 +248,7 @@ extern int nfs4_setup_sequence(const struct nfs_server *server,
> extern int nfs41_setup_sequence(struct nfs4_session *session,
> 		struct nfs4_sequence_args *args, struct nfs4_sequence_res *res,
> 		struct rpc_task *task);
> +extern void nfs41_sequence_free_slot(struct nfs4_sequence_res *res);
> extern void nfs4_destroy_session(struct nfs4_session *session);
> extern struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp);
> extern int nfs4_proc_create_session(struct nfs_client *, struct rpc_cred *);
> diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
> index 52d8472..80aa19a 100644
> --- a/fs/nfs/nfs4filelayout.c
> +++ b/fs/nfs/nfs4filelayout.c
> @@ -291,19 +291,20 @@ static void filelayout_read_prepare(struct rpc_task *task, void *data)
> {
> 	struct nfs_read_data *rdata = data;
> 
> +	if (nfs41_setup_sequence(rdata->ds_clp->cl_session,
> +				&rdata->args.seq_args, &rdata->res.seq_res,
> +				task))
> +		return;
> +
> 	if (filelayout_reset_to_mds(rdata->header->lseg)) {
> 		dprintk("%s task %u reset io to MDS\n", __func__, task->tk_pid);
> 		filelayout_reset_read(rdata);
> +		nfs41_sequence_free_slot(&rdata->res.seq_res);
> 		rpc_exit(task, 0);
> 		return;
> 	}
> 	rdata->read_done_cb = filelayout_read_done_cb;
> 
> -	if (nfs41_setup_sequence(rdata->ds_clp->cl_session,
> -				&rdata->args.seq_args, &rdata->res.seq_res,
> -				task))
> -		return;
> -
> 	rpc_call_start(task);
> }
> 
> @@ -393,17 +394,18 @@ static void filelayout_write_prepare(struct rpc_task *task, void *data)
> {
> 	struct nfs_write_data *wdata = data;
> 
> +	if (nfs41_setup_sequence(wdata->ds_clp->cl_session,
> +				&wdata->args.seq_args, &wdata->res.seq_res,
> +				task))
> +		return;
> +
> 	if (filelayout_reset_to_mds(wdata->header->lseg)) {
> 		dprintk("%s task %u reset io to MDS\n", __func__, task->tk_pid);
> 		filelayout_reset_write(wdata);
> +		nfs41_sequence_free_slot(&rdata->res.seq_res);
> 		rpc_exit(task, 0);
> 		return;
> 	}
> -	if (nfs41_setup_sequence(wdata->ds_clp->cl_session,
> -				&wdata->args.seq_args, &wdata->res.seq_res,
> -				task))
> -		return;
> -
> 	rpc_call_start(task);
> }
> 
> diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
> index 68b21d8..d186be2 100644
> --- a/fs/nfs/nfs4proc.c
> +++ b/fs/nfs/nfs4proc.c
> @@ -468,7 +468,7 @@ void nfs4_check_drain_bc_complete(struct nfs4_session *ses)
> 	complete(&ses->bc_slot_table.complete);
> }
> 
> -static void nfs41_sequence_free_slot(struct nfs4_sequence_res *res)
> +void nfs41_sequence_free_slot(struct nfs4_sequence_res *res)
> {
> 	struct nfs4_slot_table *tbl;
> 
> @@ -486,6 +486,7 @@ static void nfs41_sequence_free_slot(struct nfs4_sequence_res *res)
> 	spin_unlock(&tbl->slot_tbl_lock);
> 	res->sr_slot = NULL;
> }
> +EXPORT_SYMBOL_GPL(nfs41_sequence_free_slot);
> 
> static int nfs41_sequence_done(struct rpc_task *task, struct nfs4_sequence_res *res)
> {
> 
> -- 
> Trond Myklebust
> Linux NFS client maintainer
> 
> NetApp
> Trond.Myklebust@netapp.com
> www.netapp.com