[PATCH 0/5] Allow parallel opens in NFSv4.1

[PATCH 0/5] Allow parallel opens in NFSv4.1

[Trond Myklebust]

The following patch set converts the sequence id locking code to allow
parallel OPEN calls that are serialised with OPEN_DOWNGRADE and CLOSE.

Trond Myklebust (5):
  NFSv4: Don't update the open stateid unless it is newer than the old
    one
  NFSv4: Refactor NFS sequence id counter
  NFSv4: Convert NFS sequence id lock into a shared/exclusive lock
  NFSv4.1: Allow parallel OPEN requests
  NFSv4: Use correct locking when updating nfs4_state in nfs4_close_done

 fs/nfs/nfs4_fs.h   |  32 ++++++++++--
 fs/nfs/nfs4proc.c  |  84 ++++++++++++++++++------------
 fs/nfs/nfs4state.c | 150 +++++++++++++++++++++++++++++++++++++++++------------
 3 files changed, 195 insertions(+), 71 deletions(-)

-- 
1.8.5.3

[PATCH 1/5] NFSv4: Don't update the open stateid unless it is newer than the old one

[Trond Myklebust]

This patch is in preparation for the NFSv4.1 parallel open capability.

Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>
---
 fs/nfs/nfs4_fs.h  | 10 ++++++++++
 fs/nfs/nfs4proc.c | 21 +++++++++++++++++----
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index a5b27c2d9689..df81fcc138a7 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -500,6 +500,16 @@ static inline bool nfs4_stateid_match(const nfs4_stateid *dst, const nfs4_statei
 	return memcmp(dst, src, sizeof(*dst)) == 0;
 }
 
+static inline bool nfs4_stateid_match_other(const nfs4_stateid *dst, const nfs4_stateid *src)
+{
+	return memcmp(dst->other, src->other, NFS4_STATEID_OTHER_SIZE) == 0;
+}
+
+static inline bool nfs4_stateid_is_newer(const nfs4_stateid *s1, const nfs4_stateid *s2)
+{
+	return (s32)(be32_to_cpu(s1->seqid) - be32_to_cpu(s2->seqid)) > 0;
+}
+
 static inline bool nfs4_valid_open_stateid(const struct nfs4_state *state)
 {
 	return test_bit(NFS_STATE_RECOVERY_FAILED, &state->flags) == 0;
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 2da6a698b8f7..96e0bd42f38c 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -1137,12 +1137,20 @@ static void update_open_stateflags(struct nfs4_state *state, fmode_t fmode)
 	nfs4_state_set_mode_locked(state, state->state | fmode);
 }
 
+static bool nfs_need_update_open_stateid(struct nfs4_state *state,
+		nfs4_stateid *stateid)
+{
+	if (test_and_set_bit(NFS_OPEN_STATE, &state->flags) == 0)
+		return true;
+	if (!nfs4_stateid_match_other(stateid, &state->open_stateid))
+		return true;
+	if (nfs4_stateid_is_newer(stateid, &state->open_stateid))
+		return true;
+	return false;
+}
+
 static void nfs_set_open_stateid_locked(struct nfs4_state *state, nfs4_stateid *stateid, fmode_t fmode)
 {
-	if (test_bit(NFS_DELEGATED_STATE, &state->flags) == 0)
-		nfs4_stateid_copy(&state->stateid, stateid);
-	nfs4_stateid_copy(&state->open_stateid, stateid);
-	set_bit(NFS_OPEN_STATE, &state->flags);
 	switch (fmode) {
 		case FMODE_READ:
 			set_bit(NFS_O_RDONLY_STATE, &state->flags);
@@ -1153,6 +1161,11 @@ static void nfs_set_open_stateid_locked(struct nfs4_state *state, nfs4_stateid *
 		case FMODE_READ|FMODE_WRITE:
 			set_bit(NFS_O_RDWR_STATE, &state->flags);
 	}
+	if (!nfs_need_update_open_stateid(state, stateid))
+		return;
+	if (test_bit(NFS_DELEGATED_STATE, &state->flags) == 0)
+		nfs4_stateid_copy(&state->stateid, stateid);
+	nfs4_stateid_copy(&state->open_stateid, stateid);
 }
 
 static void nfs_set_open_stateid(struct nfs4_state *state, nfs4_stateid *stateid, fmode_t fmode)
-- 
1.8.5.3

[PATCH 2/5] NFSv4: Refactor NFS sequence id counter

[Trond Myklebust]

More preparatory work for the NFSv4.1 parallel OPEN capability.

Refactor the code that deals with sequence ids to separate out the
bits that deal with serialisation, and convert those into separate
locking structures.

Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>
---
 fs/nfs/nfs4_fs.h   |  18 ++++++---
 fs/nfs/nfs4state.c | 116 +++++++++++++++++++++++++++++++++++++----------------
 2 files changed, 95 insertions(+), 39 deletions(-)

diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index df81fcc138a7..cd24d9d79830 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -63,21 +63,29 @@ struct nfs4_minor_version_ops {
 	const struct nfs4_mig_recovery_ops *mig_recovery_ops;
 };
 
+struct nfs4_stateid_lock {
+	spinlock_t lock;		/* Protects the list */
+	struct list_head list;		/* Defines sequence of RPC calls */
+	struct rpc_wait_queue	wait;	/* RPC call delay queue */
+};
+
+struct nfs4_stateid_lock_wait {
+	struct list_head list;
+	struct rpc_task *task;
+};
+
 #define NFS_SEQID_CONFIRMED 1
 struct nfs_seqid_counter {
 	ktime_t create_time;
 	int owner_id;
 	int flags;
 	u32 counter;
-	spinlock_t lock;		/* Protects the list */
-	struct list_head list;		/* Defines sequence of RPC calls */
-	struct rpc_wait_queue	wait;	/* RPC call delay queue */
+	struct nfs4_stateid_lock st_lock;
 };
 
 struct nfs_seqid {
 	struct nfs_seqid_counter *sequence;
-	struct list_head list;
-	struct rpc_task *task;
+	struct nfs4_stateid_lock_wait wait;
 };
 
 static inline void nfs_confirm_seqid(struct nfs_seqid_counter *seqid, int status)
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index e5be72518bd7..73c7c9ce6f75 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -437,20 +437,91 @@ nfs4_remove_state_owner_locked(struct nfs4_state_owner *sp)
 }
 
 static void
+nfs4_init_stateid_lock(struct nfs4_stateid_lock *st_lock)
+{
+	spin_lock_init(&st_lock->lock);
+	INIT_LIST_HEAD(&st_lock->list);
+	rpc_init_wait_queue(&st_lock->wait, "Seqid_waitqueue");
+}
+
+static void
+nfs4_destroy_stateid_lock(struct nfs4_stateid_lock *st_lock)
+{
+	rpc_destroy_wait_queue(&st_lock->wait);
+}
+
+static bool
+nfs4_stateid_lock_is_blocked(struct nfs4_stateid_lock *st_lock)
+{
+	return !list_empty(&st_lock->list);
+}
+
+static void
+nfs4_stateid_lock_wake_next_locked(struct nfs4_stateid_lock *st_lock)
+{
+	struct nfs4_stateid_lock_wait *next;
+
+	if (list_empty(&st_lock->list))
+		return;
+
+	next = list_first_entry(&st_lock->list,
+				struct nfs4_stateid_lock_wait,
+				list);
+	rpc_wake_up_queued_task(&st_lock->wait, next->task);
+}
+
+static void
+nfs4_stateid_lock_wait_init(struct nfs4_stateid_lock_wait *wait)
+{
+	INIT_LIST_HEAD(&wait->list);
+	wait->task = NULL;
+}
+
+static int
+nfs4_stateid_lock_wait_begin(struct nfs4_stateid_lock *st_lock,
+		struct nfs4_stateid_lock_wait *wait,
+		struct rpc_task *task)
+{
+	int status = 0;
+
+	spin_lock(&st_lock->lock);
+	wait->task = task;
+	if (list_empty(&wait->list))
+		list_add_tail(&wait->list, &st_lock->list);
+	if (list_first_entry(&st_lock->list, struct nfs4_stateid_lock_wait, list) == wait)
+		goto unlock;
+	rpc_sleep_on(&st_lock->wait, task, NULL);
+	status = -EAGAIN;
+unlock:
+	spin_unlock(&st_lock->lock);
+	return status;
+}
+
+static void
+nfs4_stateid_lock_wait_end(struct nfs4_stateid_lock *st_lock,
+		struct nfs4_stateid_lock_wait *wait)
+{
+	if (list_empty(&wait->list))
+		return;
+	spin_lock(&st_lock->lock);
+	list_del_init(&wait->list);
+	nfs4_stateid_lock_wake_next_locked(st_lock);
+	spin_unlock(&st_lock->lock);
+}
+
+static void
 nfs4_init_seqid_counter(struct nfs_seqid_counter *sc)
 {
 	sc->create_time = ktime_get();
 	sc->flags = 0;
 	sc->counter = 0;
-	spin_lock_init(&sc->lock);
-	INIT_LIST_HEAD(&sc->list);
-	rpc_init_wait_queue(&sc->wait, "Seqid_waitqueue");
+	nfs4_init_stateid_lock(&sc->st_lock);
 }
 
 static void
 nfs4_destroy_seqid_counter(struct nfs_seqid_counter *sc)
 {
-	rpc_destroy_wait_queue(&sc->wait);
+	nfs4_destroy_stateid_lock(&sc->st_lock);
 }
 
 /*
@@ -975,7 +1046,7 @@ static int nfs4_copy_lock_stateid(nfs4_stateid *dst,
 		nfs4_stateid_copy(dst, &lsp->ls_stateid);
 		ret = 0;
 		smp_rmb();
-		if (!list_empty(&lsp->ls_seqid.list))
+		if (nfs4_stateid_lock_is_blocked(&lsp->ls_seqid.st_lock))
 			ret = -EWOULDBLOCK;
 	}
 	spin_unlock(&state->state_lock);
@@ -998,7 +1069,7 @@ static int nfs4_copy_open_stateid(nfs4_stateid *dst, struct nfs4_state *state)
 		nfs4_stateid_copy(dst, src);
 		ret = 0;
 		smp_rmb();
-		if (!list_empty(&state->owner->so_seqid.list))
+		if (nfs4_stateid_lock_is_blocked(&state->owner->so_seqid.st_lock))
 			ret = -EWOULDBLOCK;
 	} while (read_seqretry(&state->seqlock, seq));
 	return ret;
@@ -1037,29 +1108,16 @@ struct nfs_seqid *nfs_alloc_seqid(struct nfs_seqid_counter *counter, gfp_t gfp_m
 	new = kmalloc(sizeof(*new), gfp_mask);
 	if (new != NULL) {
 		new->sequence = counter;
-		INIT_LIST_HEAD(&new->list);
-		new->task = NULL;
+		nfs4_stateid_lock_wait_init(&new->wait);
 	}
 	return new;
 }
 
 void nfs_release_seqid(struct nfs_seqid *seqid)
 {
-	struct nfs_seqid_counter *sequence;
+	struct nfs_seqid_counter *sequence = seqid->sequence;
 
-	if (list_empty(&seqid->list))
-		return;
-	sequence = seqid->sequence;
-	spin_lock(&sequence->lock);
-	list_del_init(&seqid->list);
-	if (!list_empty(&sequence->list)) {
-		struct nfs_seqid *next;
-
-		next = list_first_entry(&sequence->list,
-				struct nfs_seqid, list);
-		rpc_wake_up_queued_task(&sequence->wait, next->task);
-	}
-	spin_unlock(&sequence->lock);
+	nfs4_stateid_lock_wait_end(&sequence->st_lock, &seqid->wait);
 }
 
 void nfs_free_seqid(struct nfs_seqid *seqid)
@@ -1126,19 +1184,9 @@ void nfs_increment_lock_seqid(int status, struct nfs_seqid *seqid)
 int nfs_wait_on_sequence(struct nfs_seqid *seqid, struct rpc_task *task)
 {
 	struct nfs_seqid_counter *sequence = seqid->sequence;
-	int status = 0;
 
-	spin_lock(&sequence->lock);
-	seqid->task = task;
-	if (list_empty(&seqid->list))
-		list_add_tail(&seqid->list, &sequence->list);
-	if (list_first_entry(&sequence->list, struct nfs_seqid, list) == seqid)
-		goto unlock;
-	rpc_sleep_on(&sequence->wait, task, NULL);
-	status = -EAGAIN;
-unlock:
-	spin_unlock(&sequence->lock);
-	return status;
+	return nfs4_stateid_lock_wait_begin(&sequence->st_lock,
+			&seqid->wait, task);
 }
 
 static int nfs4_run_state_manager(void *);
-- 
1.8.5.3

[PATCH 3/5] NFSv4: Convert NFS sequence id lock into a shared/exclusive lock

[Trond Myklebust]

For NFSv4.1, we want OPEN to be parallelised, while OPEN_DOWNGRADE/CLOSE
needs to be fully serialised with those OPENs. Do so by adding a
'shared lock' mode to the sequence ids.

Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>
---
 fs/nfs/nfs4_fs.h   |  6 ++++-
 fs/nfs/nfs4state.c | 64 +++++++++++++++++++++++++++++++++++++++++-------------
 2 files changed, 54 insertions(+), 16 deletions(-)

diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h
index cd24d9d79830..04ca08dada68 100644
--- a/fs/nfs/nfs4_fs.h
+++ b/fs/nfs/nfs4_fs.h
@@ -65,13 +65,17 @@ struct nfs4_minor_version_ops {
 
 struct nfs4_stateid_lock {
 	spinlock_t lock;		/* Protects the list */
-	struct list_head list;		/* Defines sequence of RPC calls */
+	int n_active;			/* Number of active shared locks */
+					/* == -1 for an exclusive lock */
+	struct list_head list;		/* Waiting RPC calls */
 	struct rpc_wait_queue	wait;	/* RPC call delay queue */
 };
 
 struct nfs4_stateid_lock_wait {
 	struct list_head list;
 	struct rpc_task *task;
+	unsigned char active : 1,
+		      shared : 1;
 };
 
 #define NFS_SEQID_CONFIRMED 1
diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c
index 73c7c9ce6f75..d1fe275b6a36 100644
--- a/fs/nfs/nfs4state.c
+++ b/fs/nfs/nfs4state.c
@@ -441,6 +441,7 @@ nfs4_init_stateid_lock(struct nfs4_stateid_lock *st_lock)
 {
 	spin_lock_init(&st_lock->lock);
 	INIT_LIST_HEAD(&st_lock->list);
+	st_lock->n_active = 0;
 	rpc_init_wait_queue(&st_lock->wait, "Seqid_waitqueue");
 }
 
@@ -453,7 +454,25 @@ nfs4_destroy_stateid_lock(struct nfs4_stateid_lock *st_lock)
 static bool
 nfs4_stateid_lock_is_blocked(struct nfs4_stateid_lock *st_lock)
 {
-	return !list_empty(&st_lock->list);
+	return st_lock->n_active != 0;
+}
+
+static bool
+nfs4_stateid_lock_make_active(struct nfs4_stateid_lock *st_lock,
+		struct nfs4_stateid_lock_wait *wait)
+{
+	if (wait->shared) {
+		if (st_lock->n_active < 0)
+			return false;
+		st_lock->n_active++;
+	} else {
+		if (st_lock->n_active != 0)
+			return false;
+		st_lock->n_active = -1;
+	}
+	list_del_init(&wait->list);
+	wait->active = 1;
+	return true;
 }
 
 static void
@@ -461,13 +480,17 @@ nfs4_stateid_lock_wake_next_locked(struct nfs4_stateid_lock *st_lock)
 {
 	struct nfs4_stateid_lock_wait *next;
 
-	if (list_empty(&st_lock->list))
-		return;
+	for (;;) {
+		if (list_empty(&st_lock->list))
+			break;
 
-	next = list_first_entry(&st_lock->list,
+		next = list_first_entry(&st_lock->list,
 				struct nfs4_stateid_lock_wait,
 				list);
-	rpc_wake_up_queued_task(&st_lock->wait, next->task);
+		if (!nfs4_stateid_lock_make_active(st_lock, next))
+			break;
+		rpc_wake_up_queued_task(&st_lock->wait, next->task);
+	}
 }
 
 static void
@@ -475,6 +498,8 @@ nfs4_stateid_lock_wait_init(struct nfs4_stateid_lock_wait *wait)
 {
 	INIT_LIST_HEAD(&wait->list);
 	wait->task = NULL;
+	wait->active = 0;
+	wait->shared = 0;
 }
 
 static int
@@ -485,14 +510,16 @@ nfs4_stateid_lock_wait_begin(struct nfs4_stateid_lock *st_lock,
 	int status = 0;
 
 	spin_lock(&st_lock->lock);
-	wait->task = task;
-	if (list_empty(&wait->list))
-		list_add_tail(&wait->list, &st_lock->list);
-	if (list_first_entry(&st_lock->list, struct nfs4_stateid_lock_wait, list) == wait)
-		goto unlock;
-	rpc_sleep_on(&st_lock->wait, task, NULL);
-	status = -EAGAIN;
-unlock:
+	if (!wait->active && list_empty(&wait->list)) {
+		wait->task = task;
+		if (!list_empty(&st_lock->list) ||
+		    !nfs4_stateid_lock_make_active(st_lock, wait))
+			list_add_tail(&wait->list, &st_lock->list);
+	}
+	if (!wait->active) {
+		rpc_sleep_on(&st_lock->wait, task, NULL);
+		status = -EAGAIN;
+	}
 	spin_unlock(&st_lock->lock);
 	return status;
 }
@@ -501,12 +528,19 @@ static void
 nfs4_stateid_lock_wait_end(struct nfs4_stateid_lock *st_lock,
 		struct nfs4_stateid_lock_wait *wait)
 {
-	if (list_empty(&wait->list))
+	if (!wait->active && list_empty(&wait->list))
 		return;
 	spin_lock(&st_lock->lock);
 	list_del_init(&wait->list);
-	nfs4_stateid_lock_wake_next_locked(st_lock);
+	if (wait->active) {
+		if (wait->shared)
+			st_lock->n_active--;
+		else
+			st_lock->n_active = 0;
+		nfs4_stateid_lock_wake_next_locked(st_lock);
+	}
 	spin_unlock(&st_lock->lock);
+	wait->active = 0;
 }
 
 static void
-- 
1.8.5.3

[PATCH 4/5] NFSv4.1: Allow parallel OPEN requests

[Trond Myklebust]

Enable parallel OPEN requests for the case of NFSv4.1. Please note that
OPENs are still completely serialised w.r.t. OPEN_DOWNGRADE and CLOSE.

Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>
---
 fs/nfs/nfs4proc.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 96e0bd42f38c..4eb78f6c3623 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -1006,6 +1006,8 @@ static struct nfs4_opendata *nfs4_opendata_alloc(struct dentry *dentry,
 	p->o_arg.clientid = server->nfs_client->cl_clientid;
 	p->o_arg.id.create_time = ktime_to_ns(sp->so_seqid.create_time);
 	p->o_arg.id.uniquifier = sp->so_seqid.owner_id;
+	if (server->caps & NFS_CAP_STATEID_NFSV41)
+		p->o_arg.seqid->wait.shared = 1;
 	p->o_arg.name = &dentry->d_name;
 	p->o_arg.server = server;
 	p->o_arg.bitmask = nfs4_bitmask(server, label);
-- 
1.8.5.3

[PATCH 5/5] NFSv4: Use correct locking when updating nfs4_state in nfs4_close_done

[Trond Myklebust]

The stateid and state->flags should be updated atomically under
protection of the state->seqlock.

Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>
---
 fs/nfs/nfs4proc.c | 65 ++++++++++++++++++++++++++++++-------------------------
 1 file changed, 35 insertions(+), 30 deletions(-)

diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 4eb78f6c3623..c0015c2d2ac2 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -1151,6 +1151,38 @@ static bool nfs_need_update_open_stateid(struct nfs4_state *state,
 	return false;
 }
 
+static void nfs_clear_open_stateid_locked(struct nfs4_state *state,
+		nfs4_stateid *stateid, fmode_t fmode)
+{
+	clear_bit(NFS_O_RDWR_STATE, &state->flags);
+	switch (fmode & (FMODE_READ|FMODE_WRITE)) {
+	case FMODE_WRITE:
+		clear_bit(NFS_O_RDONLY_STATE, &state->flags);
+		break;
+	case FMODE_READ:
+		clear_bit(NFS_O_WRONLY_STATE, &state->flags);
+		break;
+	case 0:
+		clear_bit(NFS_O_RDONLY_STATE, &state->flags);
+		clear_bit(NFS_O_WRONLY_STATE, &state->flags);
+		clear_bit(NFS_OPEN_STATE, &state->flags);
+	}
+	if (stateid == NULL)
+		return;
+	if (!nfs_need_update_open_stateid(state, stateid))
+		return;
+	if (test_bit(NFS_DELEGATED_STATE, &state->flags) == 0)
+		nfs4_stateid_copy(&state->stateid, stateid);
+	nfs4_stateid_copy(&state->open_stateid, stateid);
+}
+
+static void nfs_clear_open_stateid(struct nfs4_state *state, nfs4_stateid *stateid, fmode_t fmode)
+{
+	write_seqlock(&state->seqlock);
+	nfs_clear_open_stateid_locked(state, stateid, fmode);
+	write_sequnlock(&state->seqlock);
+}
+
 static void nfs_set_open_stateid_locked(struct nfs4_state *state, nfs4_stateid *stateid, fmode_t fmode)
 {
 	switch (fmode) {
@@ -1170,13 +1202,6 @@ static void nfs_set_open_stateid_locked(struct nfs4_state *state, nfs4_stateid *
 	nfs4_stateid_copy(&state->open_stateid, stateid);
 }
 
-static void nfs_set_open_stateid(struct nfs4_state *state, nfs4_stateid *stateid, fmode_t fmode)
-{
-	write_seqlock(&state->seqlock);
-	nfs_set_open_stateid_locked(state, stateid, fmode);
-	write_sequnlock(&state->seqlock);
-}
-
 static void __update_open_stateid(struct nfs4_state *state, nfs4_stateid *open_stateid, const nfs4_stateid *deleg_stateid, fmode_t fmode)
 {
 	/*
@@ -2491,26 +2516,6 @@ static void nfs4_free_closedata(void *data)
 	kfree(calldata);
 }
 
-static void nfs4_close_clear_stateid_flags(struct nfs4_state *state,
-		fmode_t fmode)
-{
-	spin_lock(&state->owner->so_lock);
-	clear_bit(NFS_O_RDWR_STATE, &state->flags);
-	switch (fmode & (FMODE_READ|FMODE_WRITE)) {
-	case FMODE_WRITE:
-		clear_bit(NFS_O_RDONLY_STATE, &state->flags);
-		break;
-	case FMODE_READ:
-		clear_bit(NFS_O_WRONLY_STATE, &state->flags);
-		break;
-	case 0:
-		clear_bit(NFS_O_RDONLY_STATE, &state->flags);
-		clear_bit(NFS_O_WRONLY_STATE, &state->flags);
-		clear_bit(NFS_OPEN_STATE, &state->flags);
-	}
-	spin_unlock(&state->owner->so_lock);
-}
-
 static void nfs4_close_done(struct rpc_task *task, void *data)
 {
 	struct nfs4_closedata *calldata = data;
@@ -2529,9 +2534,9 @@ static void nfs4_close_done(struct rpc_task *task, void *data)
 			if (calldata->roc)
 				pnfs_roc_set_barrier(state->inode,
 						     calldata->roc_barrier);
-			nfs_set_open_stateid(state, &calldata->res.stateid, 0);
+			nfs_clear_open_stateid(state, &calldata->res.stateid, 0);
 			renew_lease(server, calldata->timestamp);
-			break;
+			goto out_release;
 		case -NFS4ERR_ADMIN_REVOKED:
 		case -NFS4ERR_STALE_STATEID:
 		case -NFS4ERR_OLD_STATEID:
@@ -2545,7 +2550,7 @@ static void nfs4_close_done(struct rpc_task *task, void *data)
 				goto out_release;
 			}
 	}
-	nfs4_close_clear_stateid_flags(state, calldata->arg.fmode);
+	nfs_clear_open_stateid(state, NULL, calldata->arg.fmode);
 out_release:
 	nfs_release_seqid(calldata->arg.seqid);
 	nfs_refresh_inode(calldata->inode, calldata->res.fattr);
-- 
1.8.5.3