[PATCH Version 2 0/1] NFSv4.1 Fix umount when filelayout DS is also the MDS

[PATCH Version 2 0/1] NFSv4.1 Fix umount when filelayout DS is also the MDS

[andros]

From: Andy Adamson <andros@netapp.com>

Version 2: Corrected CONFIG_NFS issues.

At the Bakeathon, Jorge noted that when the MDS and DS use the same struct
nfs_client, umount would fail to free the nfs_client struct and the
keep-alive SEQUENCE compound would continue ad infinitum.

This was due to the fact that the DS reference to the MDS nfs_client would
prevent the umount from dropping the cl_count to zero. Moreover, the
DS refererence is only dropped by the deviceid dereference only called
by nfs4_deviceid_purge_client from nfs_free_client when the nfs_client
cl_count was zero.

See the patch comments for the solution description.

I've tested this solution against a 2-node C-mode filer where one node is an
MDS/DS and the other node a standalone MDS but uses the first node as a DS.

I tested all combinations, including mounting the MDS/DS node, and using pNFS
for I/O. Then mounting the solo MDS node, and using pNFS for I/O (which uses
the MDS/DS node as a DS), then umounting one of the mount points, doing pNFS
I/O and then umount the other.

In all cases, umount destroyed the appropriate nfs_client, and associated
session.


Andy Adamson (1):
  NFSv4.1 Fix umount when filelayout DS is also the MDS

 fs/nfs/client.c            |   88 ++++++++++++++++++++++++++++++++++++++++++-
 fs/nfs/internal.h          |    1 +
 fs/nfs/nfs4filelayoutdev.c |   29 ++++++++++++--
 include/linux/nfs_fs_sb.h  |    1 +
 4 files changed, 112 insertions(+), 7 deletions(-)

-- 
1.7.6.4

[PATCH Version 2 1/1] NFSv4.1 Fix umount when filelayout DS is also the MDS

[andros]

From: Andy Adamson <andros@netapp.com>

Add a secondary creation count to struct nfs_client to handle the corner case
when a file layout data server is also the mounted MDS, and shares the
cl_session. In this case, a umount of the MDS should not destroy the nfs_client
as it could still be in use as a DS (only) for another deviceid/MDS.

Currently there is a 'chicken and egg' issue when the DS is also the mounted
MDS. The nfs_match_client() reference from nfs4_set_ds_client bumps the
cl_count, the nfs_client is not freed at umount, and nfs4_deviceid_purge_client
is not called to dereference the MDS usage of a deviceid which holds a
reference to the DS nfs_client.  The result is the umount program returns,
but the nfs_client is not freed, and the cl_session hearbeat continues.

The MDS (and all other nfs mounts) lose their last nfs_client reference in
nfs_free_server when the last nfs_server (fsid) is umounted.
The file layout DS lose their last nfs_client reference in destroy_ds
when the last deviceid referencing the data server is put and destroy_ds is
called. This is triggered by a call to nfs4_deviceid_purge_client which
removes references to a pNFS deviceid used by an MDS mount.

The new cl_ds_count is an additional 'creation' reference for a file layout
data server struct nfs_client. When an nfs_client is a DS,
the cl_ds_count is incremented from 0 to 1, and decremented from 1 to zero
in destroy_ds called on the last deviceid reference to the data server.

Both the cl_count and the cl_ds_count must be zero to free the nfs_client.

With the cl_ds_count, when the DS is also an MDS, the nfs_match_client
reference from nfs4_set_ds_client triggered by the DS finding the existing
MDS nfs_client is dropped so that the umount call to nfs_free_server
dereferences the (MDS) cl_count to 0. But since the cl_ds_count
is 1, the nfs_client struct is not freed.  Instead, nfs4_deviceid_purge_client
is called to dereference the umounted MDS deviceids. The deviceid in
turn dereferences the data server.

When the DS is not the mounted MDS, the final nfs_client cl_count reference is
dropped in destroy_ds.

Reported-by: Jorge Mora <jorge.mora@netapp.com>
Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfs/client.c            |   88 ++++++++++++++++++++++++++++++++++++++++++-
 fs/nfs/internal.h          |    1 +
 fs/nfs/nfs4filelayoutdev.c |   29 ++++++++++++--
 include/linux/nfs_fs_sb.h  |    1 +
 4 files changed, 112 insertions(+), 7 deletions(-)

diff --git a/fs/nfs/client.c b/fs/nfs/client.c
index 17ba6b9..3b4f981 100644
--- a/fs/nfs/client.c
+++ b/fs/nfs/client.c
@@ -159,6 +159,7 @@ static struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_
 	clp->rpc_ops = cl_init->rpc_ops;
 
 	atomic_set(&clp->cl_count, 1);
+	atomic_set(&clp->cl_ds_count, 0);
 	clp->cl_cons_state = NFS_CS_INITING;
 
 	memcpy(&clp->cl_addr, cl_init->addr, cl_init->addrlen);
@@ -213,10 +214,25 @@ static void nfs4_shutdown_session(struct nfs_client *clp)
 	}
 
 }
+
+/*
+ * Call nfs4_deviceid_purge client to dereference this MDS's use of
+ * deviceids which in turn dereference DSs. The nfs_client will be
+ * freed upon last deviceid DS reference.
+ */
+static void nfs4_shutdown_mds_ds_session(struct nfs_client *clp)
+{
+	if (nfs4_has_session(clp))
+		nfs4_deviceid_purge_client(clp);
+}
 #else /* CONFIG_NFS_V4_1 */
 static void nfs4_shutdown_session(struct nfs_client *clp)
 {
 }
+
+static void nfs4_shutdown_mds_ds_session(struct nfs_client *clp)
+{
+}
 #endif /* CONFIG_NFS_V4_1 */
 
 /*
@@ -228,6 +244,11 @@ static void nfs4_destroy_callback(struct nfs_client *clp)
 		nfs_callback_down(clp->cl_mvops->minor_version);
 }
 
+static void nfs4_shutdown_mds_ds_client(struct nfs_client *clp)
+{
+	nfs4_shutdown_mds_ds_session(clp);
+}
+
 static void nfs4_shutdown_client(struct nfs_client *clp)
 {
 	if (__test_and_clear_bit(NFS_CS_RENEWD, &clp->cl_res_state))
@@ -275,6 +296,10 @@ static void nfs4_shutdown_client(struct nfs_client *clp)
 {
 }
 
+static void nfs4_shutdown_mds_ds_client(struct nfs_client *clp)
+{
+}
+
 void nfs_cleanup_cb_ident_idr(struct net *net)
 {
 }
@@ -316,18 +341,57 @@ static void nfs_free_client(struct nfs_client *clp)
 
 /*
  * Release a reference to a shared client record
+ *
+ * A secondary creation count is added to struct nfs_client to handle the
+ * corner case when a file layout data server is also the mounted MDS, and
+ * shares the cl_session. In this case, a umount of the MDS should not destroy
+ * the nfs_client as it could still be in use as a DS (only) for another
+ * deviceid/MDS.
+ *
+ * Both cl_count and ds_cl_count must be zero to free the nfs_client.
+ *
+ * The cl_ds_count is an additional 'creation' reference for a file layout
+ * data server struct nfs_client. When an nfs_client is a DS,
+ * the cl_ds_count is incremented from 0 to 1, and decremented from 1 to zero
+ * in destroy_ds called on the last deviceid reference to the data server.
+ *
+ * With the cl_ds_count, when the DS is also an MDS, the nfs_match_client
+ * reference from nfs4_set_ds_client is dropped so that the umount call to
+ * nfs_free_server dereferences the cl_count to 0. But since the cl_ds_count
+ * is 1, the nfs_client struct is not freed. Instead,nfs4_deviceid_purge_client
+ * is called to dereference the newly umounted MDS deviceids. The deviceid in
+ * turn dereferences the data server.
  */
-void nfs_put_client(struct nfs_client *clp)
+void _nfs_put_client(struct nfs_client *clp, bool is_ds_call)
 {
 	struct nfs_net *nn;
+	atomic_t *primary = &clp->cl_count, *secondary = &clp->cl_ds_count;
 
 	if (!clp)
 		return;
 
-	dprintk("--> nfs_put_client({%d})\n", atomic_read(&clp->cl_count));
+	dprintk("--> nfs_put_client ({%d, %d})\n", atomic_read(&clp->cl_count),
+		atomic_read(&clp->cl_ds_count));
+
+	if (is_ds_call) {
+		primary = &clp->cl_ds_count;
+		secondary = &clp->cl_count;
+	}
 	nn = net_generic(clp->cl_net, nfs_net_id);
 
-	if (atomic_dec_and_lock(&clp->cl_count, &nn->nfs_client_lock)) {
+	/* Dereference the primary count and inspect the secondary count */
+	if (atomic_dec_and_lock(primary, &nn->nfs_client_lock)) {
+		if (atomic_read(secondary) != 0) {
+			spin_unlock(&nn->nfs_client_lock);
+			if (is_ds_call == false) {
+				/*
+				 * This is an MDS-DS nfs_client. The MDS
+				 * use of this nfs_client struct is complete.
+				 */
+				nfs4_shutdown_mds_ds_client(clp);
+			}
+			return;
+		}
 		list_del(&clp->cl_share_link);
 		nfs_cb_idr_remove_locked(clp);
 		spin_unlock(&nn->nfs_client_lock);
@@ -337,8 +401,19 @@ void nfs_put_client(struct nfs_client *clp)
 		nfs_free_client(clp);
 	}
 }
+
+void nfs_put_client(struct nfs_client *clp)
+{
+	_nfs_put_client(clp, false);
+}
 EXPORT_SYMBOL_GPL(nfs_put_client);
 
+void nfs_put_ds_client(struct nfs_client *clp)
+{
+	_nfs_put_client(clp, true);
+}
+EXPORT_SYMBOL_GPL(nfs_put_ds_client);
+
 #if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
 /*
  * Test if two ip6 socket addresses refer to the same socket by
@@ -1511,6 +1586,13 @@ struct nfs_client *nfs4_set_ds_client(struct nfs_client* mds_clp,
 	clp = nfs_get_client(&cl_init, &ds_timeout, mds_clp->cl_ipaddr,
 			     mds_clp->cl_rpcclient->cl_auth->au_flavor);
 
+	/*
+	 * Data servers use the cl_ds_count as an additional creation reference
+	 * for the case when a DS is also the MDS. Balanced in destroy_ds.
+	 */
+	if (!IS_ERR(clp))
+		atomic_inc(&clp->cl_ds_count);
+
 	dprintk("<-- %s %p\n", __func__, clp);
 	return clp;
 }
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 18f99ef..d31a2c1 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -151,6 +151,7 @@ extern void nfs_clients_init(struct net *net);
 
 extern void nfs_cleanup_cb_ident_idr(struct net *);
 extern void nfs_put_client(struct nfs_client *);
+extern void nfs_put_ds_client(struct nfs_client *);
 extern struct nfs_client *nfs4_find_client_ident(struct net *, int);
 extern struct nfs_client *
 nfs4_find_client_sessionid(struct net *, const struct sockaddr *,
diff --git a/fs/nfs/nfs4filelayoutdev.c b/fs/nfs/nfs4filelayoutdev.c
index f81231f..91491c9 100644
--- a/fs/nfs/nfs4filelayoutdev.c
+++ b/fs/nfs/nfs4filelayoutdev.c
@@ -190,6 +190,7 @@ nfs4_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds)
 		dprintk("%s: DS %s: trying address %s\n",
 			__func__, ds->ds_remotestr, da->da_remotestr);
 
+		/* References clp->cl_ds_count */
 		clp = nfs4_set_ds_client(mds_srv->nfs_client,
 					(struct sockaddr *)&da->da_addr,
 					da->da_addrlen, IPPROTO_TCP,
@@ -203,6 +204,17 @@ nfs4_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds)
 		goto out;
 	}
 
+	/*
+	 * If DS is the MDS drop the nfs_match_client reference which
+	 * is for mulitple nfs_servers referencing the nfs_client and is
+	 * dereferenced at each nfs_server umount.
+	 * Data servers use the cl_ds_count instead so that the nfs_client
+	 * can still serve as a DS (if so configured) after the umount
+	 * of the MDS.
+	 */
+	if (clp == mds_srv->nfs_client)
+		nfs_put_client(clp);
+
 	status = nfs4_init_ds_session(clp, mds_srv->nfs_client->cl_lease_time);
 	if (status)
 		goto out_put;
@@ -217,7 +229,7 @@ out_put:
 }
 
 static void
-destroy_ds(struct nfs4_pnfs_ds *ds)
+destroy_ds(struct nfs4_pnfs_ds *ds, const struct nfs_client *mds_clp)
 {
 	struct nfs4_pnfs_ds_addr *da;
 
@@ -225,8 +237,17 @@ destroy_ds(struct nfs4_pnfs_ds *ds)
 	ifdebug(FACILITY)
 		print_ds(ds);
 
-	if (ds->ds_clp)
-		nfs_put_client(ds->ds_clp);
+	if (ds->ds_clp) {
+		/* Drop the cl_ds_count creation reference */
+		nfs_put_ds_client(ds->ds_clp);
+		/*
+		 * If DS is not the MDS, dereference the cl_count
+		 * (dereferenced by nfs_free_server if the DS is the MDS).
+		 * Balances the inital SET of cl_count.
+		 */
+		if (ds->ds_clp != mds_clp)
+			nfs_put_client(ds->ds_clp);
+	}
 
 	while (!list_empty(&ds->ds_addrs)) {
 		da = list_first_entry(&ds->ds_addrs,
@@ -256,7 +277,7 @@ nfs4_fl_free_deviceid(struct nfs4_file_layout_dsaddr *dsaddr)
 						&nfs4_ds_cache_lock)) {
 				list_del_init(&ds->ds_node);
 				spin_unlock(&nfs4_ds_cache_lock);
-				destroy_ds(ds);
+				destroy_ds(ds, dsaddr->id_node.nfs_client);
 			}
 		}
 	}
diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
index fbb78fb..7f0304d 100644
--- a/include/linux/nfs_fs_sb.h
+++ b/include/linux/nfs_fs_sb.h
@@ -25,6 +25,7 @@ struct nfs41_impl_id;
  */
 struct nfs_client {
 	atomic_t		cl_count;
+	atomic_t		cl_ds_count;
 	int			cl_cons_state;	/* current construction state (-ve: init error) */
 #define NFS_CS_READY		0		/* ready to be used */
 #define NFS_CS_INITING		1		/* busy initialising */
-- 
1.7.6.4

Re: [PATCH Version 2 1/1] NFSv4.1 Fix umount when filelayout DS is also the MDS

[Myklebust, Trond]

T24gVGh1LCAyMDEyLTA2LTE0IGF0IDEyOjEyIC0wNDAwLCBhbmRyb3NAbmV0YXBwLmNvbSB3cm90
ZToNCg0KPiAtLS0NCj4gIGZzL25mcy9jbGllbnQuYyAgICAgICAgICAgIHwgICA4OCArKysrKysr
KysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKystDQo+ICBmcy9uZnMvaW50ZXJuYWwu
aCAgICAgICAgICB8ICAgIDEgKw0KPiAgZnMvbmZzL25mczRmaWxlbGF5b3V0ZGV2LmMgfCAgIDI5
ICsrKysrKysrKysrKy0tDQo+ICBpbmNsdWRlL2xpbnV4L25mc19mc19zYi5oICB8ICAgIDEgKw0K
PiAgNCBmaWxlcyBjaGFuZ2VkLCAxMTIgaW5zZXJ0aW9ucygrKSwgNyBkZWxldGlvbnMoLSkNCg0K
SGkgQW5keSwNCg0KSXMgdGhlcmUgYW55IHJlYXNvbiB3aHkgdGhlIGZvbGxvd2luZyB3b3VsZG4n
dCB3b3JrIGluc3RlYWQ/IEl0IGlzDQplc3NlbnRpYWxseSB0aGUgc2FtZSBpZGVhLg0KDQpDaGVl
cnMNCiAgVHJvbmQNCjg8LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLQ0KRnJvbSBkYmVlMzUxM2YxMGMzMzI0YWEwNjgzZDQxNTE5M2E1ZDJhZjZh
MzFlIE1vbiBTZXAgMTcgMDA6MDA6MDAgMjAwMQ0KRnJvbTogVHJvbmQgTXlrbGVidXN0IDxUcm9u
ZC5NeWtsZWJ1c3RAbmV0YXBwLmNvbT4NCkRhdGU6IFRodSwgMTQgSnVuIDIwMTIgMTM6MDg6Mzgg
LTA0MDANClN1YmplY3Q6IFtQQVRDSF0gTkZTdjQuMTogRml4IHVtb3VudCB3aGVuIGZpbGVsYXlv
dXQgRFMgaXMgYWxzbyB0aGUgTURTDQoNCkN1cnJlbnRseSB0aGVyZSBpcyBhICdjaGlja2VuIGFu
ZCBlZ2cnIGlzc3VlIHdoZW4gdGhlIERTIGlzIGFsc28gdGhlIG1vdW50ZWQNCk1EUy4gVGhlIG5m
c19tYXRjaF9jbGllbnQoKSByZWZlcmVuY2UgZnJvbSBuZnM0X3NldF9kc19jbGllbnQgYnVtcHMg
dGhlDQpjbF9jb3VudCwgdGhlIG5mc19jbGllbnQgaXMgbm90IGZyZWVkIGF0IHVtb3VudCwgYW5k
IG5mczRfZGV2aWNlaWRfcHVyZ2VfY2xpZW50DQppcyBub3QgY2FsbGVkIHRvIGRlcmVmZXJlbmNl
IHRoZSBNRFMgdXNhZ2Ugb2YgYSBkZXZpY2VpZCB3aGljaCBob2xkcyBhDQpyZWZlcmVuY2UgdG8g
dGhlIERTIG5mc19jbGllbnQuICBUaGUgcmVzdWx0IGlzIHRoZSB1bW91bnQgcHJvZ3JhbSByZXR1
cm5zLA0KYnV0IHRoZSBuZnNfY2xpZW50IGlzIG5vdCBmcmVlZCwgYW5kIHRoZSBjbF9zZXNzaW9u
IGhlYXJiZWF0IGNvbnRpbnVlcy4NCg0KVGhlIE1EUyAoYW5kIGFsbCBvdGhlciBuZnMgbW91bnRz
KSBsb3NlIHRoZWlyIGxhc3QgbmZzX2NsaWVudCByZWZlcmVuY2UgaW4NCm5mc19mcmVlX3NlcnZl
ciB3aGVuIHRoZSBsYXN0IG5mc19zZXJ2ZXIgKGZzaWQpIGlzIHVtb3VudGVkLg0KVGhlIGZpbGUg
bGF5b3V0IERTIGxvc2UgdGhlaXIgbGFzdCBuZnNfY2xpZW50IHJlZmVyZW5jZSBpbiBkZXN0cm95
X2RzDQp3aGVuIHRoZSBsYXN0IGRldmljZWlkIHJlZmVyZW5jaW5nIHRoZSBkYXRhIHNlcnZlciBp
cyBwdXQgYW5kIGRlc3Ryb3lfZHMgaXMNCmNhbGxlZC4gVGhpcyBpcyB0cmlnZ2VyZWQgYnkgYSBj
YWxsIHRvIG5mczRfZGV2aWNlaWRfcHVyZ2VfY2xpZW50IHdoaWNoDQpyZW1vdmVzIHJlZmVyZW5j
ZXMgdG8gYSBwTkZTIGRldmljZWlkIHVzZWQgYnkgYW4gTURTIG1vdW50Lg0KDQpUaGUgZml4IGlz
IHRvIHRyYWNrIGhvdyBtYW55IGZpbGVzeXN0ZW1zIGFyZSBtb3VudGVkIGZyb20gdGhpcyBzZXJ2
ZXIsDQphbmQgdGhlbiB0byBwdXJnZSB0aGUgZGV2aWNlIGlkIGNhY2hlIG9uY2UgdGhhdCBjb3Vu
dCBpcyB6ZXJvLg0KDQpSZXBvcnRlZC1ieTogSm9yZ2UgTW9yYSA8Sm9yZ2UuTW9yYUBuZXRhcHAu
Y29tPg0KUmVwb3J0ZWQtYnk6IEFuZHkgQWRhbXNvbiA8YW5kcm9zQG5ldGFwcC5jb20+DQpTaWdu
ZWQtb2ZmLWJ5OiBUcm9uZCBNeWtsZWJ1c3QgPFRyb25kLk15a2xlYnVzdEBuZXRhcHAuY29tPg0K
LS0tDQogZnMvbmZzL2NsaWVudC5jICAgICAgICAgICB8ICAgNDMgKysrKysrKysrKysrKysrKysr
KysrKysrKysrKysrKysrKysrKy0tLS0tLQ0KIGluY2x1ZGUvbGludXgvbmZzX2ZzX3NiLmggfCAg
ICAxICsNCiAyIGZpbGVzIGNoYW5nZWQsIDM4IGluc2VydGlvbnMoKyksIDYgZGVsZXRpb25zKC0p
DQoNCmRpZmYgLS1naXQgYS9mcy9uZnMvY2xpZW50LmMgYi9mcy9uZnMvY2xpZW50LmMNCmluZGV4
IDE3YmE2YjkuLmZhYTU4ZDUgMTAwNjQ0DQotLS0gYS9mcy9uZnMvY2xpZW50LmMNCisrKyBiL2Zz
L25mcy9jbGllbnQuYw0KQEAgLTIwNywxNiArMjA3LDI1IEBAIGVycm9yXzA6DQogc3RhdGljIHZv
aWQgbmZzNF9zaHV0ZG93bl9zZXNzaW9uKHN0cnVjdCBuZnNfY2xpZW50ICpjbHApDQogew0KIAlp
ZiAobmZzNF9oYXNfc2Vzc2lvbihjbHApKSB7DQotCQluZnM0X2RldmljZWlkX3B1cmdlX2NsaWVu
dChjbHApOw0KIAkJbmZzNF9kZXN0cm95X3Nlc3Npb24oY2xwLT5jbF9zZXNzaW9uKTsNCiAJCW5m
czRfZGVzdHJveV9jbGllbnRpZChjbHApOw0KIAl9DQogDQogfQ0KKw0KK3N0YXRpYyB2b2lkIG5m
czRfcHVyZ2VfZGF0YXNlcnZlcnMoc3RydWN0IG5mc19jbGllbnQgKmNscCkNCit7DQorCWlmIChu
ZnM0X2hhc19zZXNzaW9uKGNscCkpDQorCQluZnM0X2RldmljZWlkX3B1cmdlX2NsaWVudChjbHAp
Ow0KK30NCiAjZWxzZSAvKiBDT05GSUdfTkZTX1Y0XzEgKi8NCiBzdGF0aWMgdm9pZCBuZnM0X3No
dXRkb3duX3Nlc3Npb24oc3RydWN0IG5mc19jbGllbnQgKmNscCkNCiB7DQogfQ0KKw0KK3N0YXRp
YyB2b2lkIG5mczRfcHVyZ2VfZGF0YXNlcnZlcnMoc3RydWN0IG5mc19jbGllbnQgKmNscCkNCit7
DQorfQ0KICNlbmRpZiAvKiBDT05GSUdfTkZTX1Y0XzEgKi8NCiANCiAvKg0KQEAgLTgwMSw2ICs4
MTAsMjkgQEAgc3RhdGljIGludCBuZnNfaW5pdF9zZXJ2ZXJfcnBjY2xpZW50KHN0cnVjdCBuZnNf
c2VydmVyICpzZXJ2ZXIsDQogCXJldHVybiAwOw0KIH0NCiANCitzdGF0aWMgaW50DQorbmZzX3Nl
cnZlcl9zZXRfY2xpZW50KHN0cnVjdCBuZnNfc2VydmVyICpzZXJ2ZXIsIHN0cnVjdCBuZnNfY2xp
ZW50ICpjbHApDQorew0KKwkvKiBUcmFjayB0aGUgbnVtYmVyIG9mIGZpbGVzeXN0ZW1zIGJlaW5n
IG1vdW50ZWQgZnJvbSB0aGlzIHNlcnZlciAqLw0KKwlhdG9taWNfaW5jKCZjbHAtPmNsX3NlcnZl
cl9jb3VudCk7DQorCXNlcnZlci0+bmZzX2NsaWVudCA9IGNscDsNCisJcmV0dXJuIDA7DQorfQ0K
Kw0KK3N0YXRpYyB2b2lkDQorbmZzX3NlcnZlcl91bnNldF9jbGllbnQoc3RydWN0IG5mc19zZXJ2
ZXIgKnNlcnZlcikNCit7DQorCXN0cnVjdCBuZnNfY2xpZW50ICpjbHAgPSBzZXJ2ZXItPm5mc19j
bGllbnQ7DQorDQorCS8qIFB1cmdlIHRoZSBsaXN0IG9mIHBORlMgZGF0YSBzZXJ2ZXJzIG9uY2Ug
dGhlIG51bWJlciBvZiBtb3VudGVkDQorCSAqIGZpbGVzeXN0ZW1zIGZhbGxzIHRvIHplcm8uDQor
CSAqLw0KKwlpZiAoYXRvbWljX2RlY19hbmRfdGVzdCgmY2xwLT5jbF9zZXJ2ZXJfY291bnQpKQ0K
KwkJbmZzNF9wdXJnZV9kYXRhc2VydmVycyhjbHApOw0KKw0KKwluZnNfcHV0X2NsaWVudChjbHAp
Ow0KK30NCisNCiAvKioNCiAgKiBuZnNfaW5pdF9jbGllbnQgLSBJbml0aWFsaXNlIGFuIE5GUzIg
b3IgTkZTMyBjbGllbnQNCiAgKg0KQEAgLTg4Nyw3ICs5MTksNyBAQCBzdGF0aWMgaW50IG5mc19p
bml0X3NlcnZlcihzdHJ1Y3QgbmZzX3NlcnZlciAqc2VydmVyLA0KIAkJcmV0dXJuIFBUUl9FUlIo
Y2xwKTsNCiAJfQ0KIA0KLQlzZXJ2ZXItPm5mc19jbGllbnQgPSBjbHA7DQorCW5mc19zZXJ2ZXJf
c2V0X2NsaWVudChzZXJ2ZXIsIGNscCk7DQogDQogCS8qIEluaXRpYWxpc2UgdGhlIGNsaWVudCBy
ZXByZXNlbnRhdGlvbiBmcm9tIHRoZSBtb3VudCBkYXRhICovDQogCXNlcnZlci0+ZmxhZ3MgPSBk
YXRhLT5mbGFnczsNCkBAIC05MzQsOCArOTY2LDcgQEAgc3RhdGljIGludCBuZnNfaW5pdF9zZXJ2
ZXIoc3RydWN0IG5mc19zZXJ2ZXIgKnNlcnZlciwNCiAJcmV0dXJuIDA7DQogDQogZXJyb3I6DQot
CXNlcnZlci0+bmZzX2NsaWVudCA9IE5VTEw7DQotCW5mc19wdXRfY2xpZW50KGNscCk7DQorCW5m
c19zZXJ2ZXJfdW5zZXRfY2xpZW50KHNlcnZlcik7DQogCWRwcmludGsoIjwtLSBuZnNfaW5pdF9z
ZXJ2ZXIoKSA9IHhlcnJvciAlZFxuIiwgZXJyb3IpOw0KIAlyZXR1cm4gZXJyb3I7DQogfQ0KQEAg
LTExNDksNyArMTE4MCw3IEBAIHZvaWQgbmZzX2ZyZWVfc2VydmVyKHN0cnVjdCBuZnNfc2VydmVy
ICpzZXJ2ZXIpDQogCWlmICghSVNfRVJSKHNlcnZlci0+Y2xpZW50KSkNCiAJCXJwY19zaHV0ZG93
bl9jbGllbnQoc2VydmVyLT5jbGllbnQpOw0KIA0KLQluZnNfcHV0X2NsaWVudChzZXJ2ZXItPm5m
c19jbGllbnQpOw0KKwluZnNfc2VydmVyX3Vuc2V0X2NsaWVudChzZXJ2ZXIpOw0KIA0KIAlpZGFf
ZGVzdHJveSgmc2VydmVyLT5sb2Nrb3duZXJfaWQpOw0KIAlpZGFfZGVzdHJveSgmc2VydmVyLT5v
cGVub3duZXJfaWQpOw0KQEAgLTE0NjksNyArMTUwMCw3IEBAIHN0YXRpYyBpbnQgbmZzNF9zZXRf
Y2xpZW50KHN0cnVjdCBuZnNfc2VydmVyICpzZXJ2ZXIsDQogCSAqLw0KIAlzZXRfYml0KE5GU19D
U19DSEVDS19MRUFTRV9USU1FLCAmY2xwLT5jbF9yZXNfc3RhdGUpOw0KIA0KLQlzZXJ2ZXItPm5m
c19jbGllbnQgPSBjbHA7DQorCW5mc19zZXJ2ZXJfc2V0X2NsaWVudChzZXJ2ZXIsIGNscCk7DQog
CWRwcmludGsoIjwtLSBuZnM0X3NldF9jbGllbnQoKSA9IDAgW25ldyAlcF1cbiIsIGNscCk7DQog
CXJldHVybiAwOw0KIGVycm9yOg0KZGlmZiAtLWdpdCBhL2luY2x1ZGUvbGludXgvbmZzX2ZzX3Ni
LmggYi9pbmNsdWRlL2xpbnV4L25mc19mc19zYi5oDQppbmRleCBmYmI3OGZiLi4yZDUxMDc3IDEw
MDY0NA0KLS0tIGEvaW5jbHVkZS9saW51eC9uZnNfZnNfc2IuaA0KKysrIGIvaW5jbHVkZS9saW51
eC9uZnNfZnNfc2IuaA0KQEAgLTI1LDYgKzI1LDcgQEAgc3RydWN0IG5mczQxX2ltcGxfaWQ7DQog
ICovDQogc3RydWN0IG5mc19jbGllbnQgew0KIAlhdG9taWNfdAkJY2xfY291bnQ7DQorCWF0b21p
Y190CQljbF9zZXJ2ZXJfY291bnQ7DQogCWludAkJCWNsX2NvbnNfc3RhdGU7CS8qIGN1cnJlbnQg
Y29uc3RydWN0aW9uIHN0YXRlICgtdmU6IGluaXQgZXJyb3IpICovDQogI2RlZmluZSBORlNfQ1Nf
UkVBRFkJCTAJCS8qIHJlYWR5IHRvIGJlIHVzZWQgKi8NCiAjZGVmaW5lIE5GU19DU19JTklUSU5H
CQkxCQkvKiBidXN5IGluaXRpYWxpc2luZyAqLw0KLS0gDQoxLjcuMTAuMg0KDQoNCi0tIA0KVHJv
bmQgTXlrbGVidXN0DQpMaW51eCBORlMgY2xpZW50IG1haW50YWluZXINCg0KTmV0QXBwDQpUcm9u
ZC5NeWtsZWJ1c3RAbmV0YXBwLmNvbQ0Kd3d3Lm5ldGFwcC5jb20NCg0K

Re: [PATCH Version 2 1/1] NFSv4.1 Fix umount when filelayout DS is also the MDS

[Adamson, Andy]

On Jun 14, 2012, at 1:21 PM, Myklebust, Trond wrote:

> On Thu, 2012-06-14 at 12:12 -0400, andros@netapp.com wrote:
> 
>> ---
>> fs/nfs/client.c            |   88 ++++++++++++++++++++++++++++++++++++++++++-
>> fs/nfs/internal.h          |    1 +
>> fs/nfs/nfs4filelayoutdev.c |   29 ++++++++++++--
>> include/linux/nfs_fs_sb.h  |    1 +
>> 4 files changed, 112 insertions(+), 7 deletions(-)
> 
> Hi Andy,
> 
> Is there any reason why the following wouldn't work instead? It is
> essentially the same idea.

Hi Trond

Works great - less filling - cool!

ACK

-->Andy

> 
> Cheers
>  Trond
> 8<-------------------------------------------------------
> From dbee3513f10c3324aa0683d415193a5d2af6a31e Mon Sep 17 00:00:00 2001
> From: Trond Myklebust <Trond.Myklebust@netapp.com>
> Date: Thu, 14 Jun 2012 13:08:38 -0400
> Subject: [PATCH] NFSv4.1: Fix umount when filelayout DS is also the MDS
> 
> Currently there is a 'chicken and egg' issue when the DS is also the mounted
> MDS. The nfs_match_client() reference from nfs4_set_ds_client bumps the
> cl_count, the nfs_client is not freed at umount, and nfs4_deviceid_purge_client
> is not called to dereference the MDS usage of a deviceid which holds a
> reference to the DS nfs_client.  The result is the umount program returns,
> but the nfs_client is not freed, and the cl_session hearbeat continues.
> 
> The MDS (and all other nfs mounts) lose their last nfs_client reference in
> nfs_free_server when the last nfs_server (fsid) is umounted.
> The file layout DS lose their last nfs_client reference in destroy_ds
> when the last deviceid referencing the data server is put and destroy_ds is
> called. This is triggered by a call to nfs4_deviceid_purge_client which
> removes references to a pNFS deviceid used by an MDS mount.
> 
> The fix is to track how many filesystems are mounted from this server,
> and then to purge the device id cache once that count is zero.
> 
> Reported-by: Jorge Mora <Jorge.Mora@netapp.com>
> Reported-by: Andy Adamson <andros@netapp.com>
> Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>
> ---
> fs/nfs/client.c           |   43 +++++++++++++++++++++++++++++++++++++------
> include/linux/nfs_fs_sb.h |    1 +
> 2 files changed, 38 insertions(+), 6 deletions(-)
> 
> diff --git a/fs/nfs/client.c b/fs/nfs/client.c
> index 17ba6b9..faa58d5 100644
> --- a/fs/nfs/client.c
> +++ b/fs/nfs/client.c
> @@ -207,16 +207,25 @@ error_0:
> static void nfs4_shutdown_session(struct nfs_client *clp)
> {
> 	if (nfs4_has_session(clp)) {
> -		nfs4_deviceid_purge_client(clp);
> 		nfs4_destroy_session(clp->cl_session);
> 		nfs4_destroy_clientid(clp);
> 	}
> 
> }
> +
> +static void nfs4_purge_dataservers(struct nfs_client *clp)
> +{
> +	if (nfs4_has_session(clp))
> +		nfs4_deviceid_purge_client(clp);
> +}
> #else /* CONFIG_NFS_V4_1 */
> static void nfs4_shutdown_session(struct nfs_client *clp)
> {
> }
> +
> +static void nfs4_purge_dataservers(struct nfs_client *clp)
> +{
> +}
> #endif /* CONFIG_NFS_V4_1 */
> 
> /*
> @@ -801,6 +810,29 @@ static int nfs_init_server_rpcclient(struct nfs_server *server,
> 	return 0;
> }
> 
> +static int
> +nfs_server_set_client(struct nfs_server *server, struct nfs_client *clp)
> +{
> +	/* Track the number of filesystems being mounted from this server */
> +	atomic_inc(&clp->cl_server_count);
> +	server->nfs_client = clp;
> +	return 0;
> +}
> +
> +static void
> +nfs_server_unset_client(struct nfs_server *server)
> +{
> +	struct nfs_client *clp = server->nfs_client;
> +
> +	/* Purge the list of pNFS data servers once the number of mounted
> +	 * filesystems falls to zero.
> +	 */
> +	if (atomic_dec_and_test(&clp->cl_server_count))
> +		nfs4_purge_dataservers(clp);
> +
> +	nfs_put_client(clp);
> +}
> +
> /**
>  * nfs_init_client - Initialise an NFS2 or NFS3 client
>  *
> @@ -887,7 +919,7 @@ static int nfs_init_server(struct nfs_server *server,
> 		return PTR_ERR(clp);
> 	}
> 
> -	server->nfs_client = clp;
> +	nfs_server_set_client(server, clp);
> 
> 	/* Initialise the client representation from the mount data */
> 	server->flags = data->flags;
> @@ -934,8 +966,7 @@ static int nfs_init_server(struct nfs_server *server,
> 	return 0;
> 
> error:
> -	server->nfs_client = NULL;
> -	nfs_put_client(clp);
> +	nfs_server_unset_client(server);
> 	dprintk("<-- nfs_init_server() = xerror %d\n", error);
> 	return error;
> }
> @@ -1149,7 +1180,7 @@ void nfs_free_server(struct nfs_server *server)
> 	if (!IS_ERR(server->client))
> 		rpc_shutdown_client(server->client);
> 
> -	nfs_put_client(server->nfs_client);
> +	nfs_server_unset_client(server);
> 
> 	ida_destroy(&server->lockowner_id);
> 	ida_destroy(&server->openowner_id);
> @@ -1469,7 +1500,7 @@ static int nfs4_set_client(struct nfs_server *server,
> 	 */
> 	set_bit(NFS_CS_CHECK_LEASE_TIME, &clp->cl_res_state);
> 
> -	server->nfs_client = clp;
> +	nfs_server_set_client(server, clp);
> 	dprintk("<-- nfs4_set_client() = 0 [new %p]\n", clp);
> 	return 0;
> error:
> diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
> index fbb78fb..2d51077 100644
> --- a/include/linux/nfs_fs_sb.h
> +++ b/include/linux/nfs_fs_sb.h
> @@ -25,6 +25,7 @@ struct nfs41_impl_id;
>  */
> struct nfs_client {
> 	atomic_t		cl_count;
> +	atomic_t		cl_server_count;
> 	int			cl_cons_state;	/* current construction state (-ve: init error) */
> #define NFS_CS_READY		0		/* ready to be used */
> #define NFS_CS_INITING		1		/* busy initialising */
> -- 
> 1.7.10.2
> 
> 
> -- 
> Trond Myklebust
> Linux NFS client maintainer
> 
> NetApp
> Trond.Myklebust@netapp.com
> www.netapp.com
> 

Re: [PATCH Version 2 1/1] NFSv4.1 Fix umount when filelayout DS is also the MDS

[Myklebust, Trond]

T24gVGh1LCAyMDEyLTA2LTE0IGF0IDIxOjUyICswMDAwLCBBZGFtc29uLCBBbmR5IHdyb3RlOg0K
PiBPbiBKdW4gMTQsIDIwMTIsIGF0IDE6MjEgUE0sIE15a2xlYnVzdCwgVHJvbmQgd3JvdGU6DQo+
IA0KPiA+IE9uIFRodSwgMjAxMi0wNi0xNCBhdCAxMjoxMiAtMDQwMCwgYW5kcm9zQG5ldGFwcC5j
b20gd3JvdGU6DQo+ID4gDQo+ID4+IC0tLQ0KPiA+PiBmcy9uZnMvY2xpZW50LmMgICAgICAgICAg
ICB8ICAgODggKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrKysrLQ0KPiA+
PiBmcy9uZnMvaW50ZXJuYWwuaCAgICAgICAgICB8ICAgIDEgKw0KPiA+PiBmcy9uZnMvbmZzNGZp
bGVsYXlvdXRkZXYuYyB8ICAgMjkgKysrKysrKysrKysrLS0NCj4gPj4gaW5jbHVkZS9saW51eC9u
ZnNfZnNfc2IuaCAgfCAgICAxICsNCj4gPj4gNCBmaWxlcyBjaGFuZ2VkLCAxMTIgaW5zZXJ0aW9u
cygrKSwgNyBkZWxldGlvbnMoLSkNCj4gPiANCj4gPiBIaSBBbmR5LA0KPiA+IA0KPiA+IElzIHRo
ZXJlIGFueSByZWFzb24gd2h5IHRoZSBmb2xsb3dpbmcgd291bGRuJ3Qgd29yayBpbnN0ZWFkPyBJ
dCBpcw0KPiA+IGVzc2VudGlhbGx5IHRoZSBzYW1lIGlkZWEuDQo+IA0KPiBIaSBUcm9uZA0KPiAN
Cj4gV29ya3MgZ3JlYXQgLSBsZXNzIGZpbGxpbmcgLSBjb29sIQ0KPiANCj4gQUNLDQo+IA0KPiAt
LT5BbmR5DQoNCk1laC4uLiBJIGZvdW5kIGEgbW9yZSBjb21wYWN0IG9uZSB3aGljaCBoYXMgdGhl
IGFkZGVkIGFkdmFudGFnZSBvZiB6ZXJvDQppbXBhY3Qgb24gbm9uLXBORlMgc3lzdGVtcy4gOi0p
DQoNCkNoZWVycw0KICBUcm9uZA0KODwtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0tLS0tLS0tDQpGcm9tIDEwYmJjZGRjNWE1ZWJhMTc3MDQ5MzY3ODc1Mzg0
ZjQ2MGY1Y2FhODUgTW9uIFNlcCAxNyAwMDowMDowMCAyMDAxDQpGcm9tOiBUcm9uZCBNeWtsZWJ1
c3QgPFRyb25kLk15a2xlYnVzdEBuZXRhcHAuY29tPg0KRGF0ZTogVGh1LCAxNCBKdW4gMjAxMiAx
MzowODozOCAtMDQwMA0KU3ViamVjdDogW1BBVENIXSBORlN2NC4xOiBGaXggdW1vdW50IHdoZW4g
ZmlsZWxheW91dCBEUyBpcyBhbHNvIHRoZSBNRFMNCg0KQ3VycmVudGx5IHRoZXJlIGlzIGEgJ2No
aWNrZW4gYW5kIGVnZycgaXNzdWUgd2hlbiB0aGUgRFMgaXMgYWxzbyB0aGUgbW91bnRlZA0KTURT
LiBUaGUgbmZzX21hdGNoX2NsaWVudCgpIHJlZmVyZW5jZSBmcm9tIG5mczRfc2V0X2RzX2NsaWVu
dCBidW1wcyB0aGUNCmNsX2NvdW50LCB0aGUgbmZzX2NsaWVudCBpcyBub3QgZnJlZWQgYXQgdW1v
dW50LCBhbmQgbmZzNF9kZXZpY2VpZF9wdXJnZV9jbGllbnQNCmlzIG5vdCBjYWxsZWQgdG8gZGVy
ZWZlcmVuY2UgdGhlIE1EUyB1c2FnZSBvZiBhIGRldmljZWlkIHdoaWNoIGhvbGRzIGENCnJlZmVy
ZW5jZSB0byB0aGUgRFMgbmZzX2NsaWVudC4gIFRoZSByZXN1bHQgaXMgdGhlIHVtb3VudCBwcm9n
cmFtIHJldHVybnMsDQpidXQgdGhlIG5mc19jbGllbnQgaXMgbm90IGZyZWVkLCBhbmQgdGhlIGNs
X3Nlc3Npb24gaGVhcmJlYXQgY29udGludWVzLg0KDQpUaGUgTURTIChhbmQgYWxsIG90aGVyIG5m
cyBtb3VudHMpIGxvc2UgdGhlaXIgbGFzdCBuZnNfY2xpZW50IHJlZmVyZW5jZSBpbg0KbmZzX2Zy
ZWVfc2VydmVyIHdoZW4gdGhlIGxhc3QgbmZzX3NlcnZlciAoZnNpZCkgaXMgdW1vdW50ZWQuDQpU
aGUgZmlsZSBsYXlvdXQgRFMgbG9zZSB0aGVpciBsYXN0IG5mc19jbGllbnQgcmVmZXJlbmNlIGlu
IGRlc3Ryb3lfZHMNCndoZW4gdGhlIGxhc3QgZGV2aWNlaWQgcmVmZXJlbmNpbmcgdGhlIGRhdGEg
c2VydmVyIGlzIHB1dCBhbmQgZGVzdHJveV9kcyBpcw0KY2FsbGVkLiBUaGlzIGlzIHRyaWdnZXJl
ZCBieSBhIGNhbGwgdG8gbmZzNF9kZXZpY2VpZF9wdXJnZV9jbGllbnQgd2hpY2gNCnJlbW92ZXMg
cmVmZXJlbmNlcyB0byBhIHBORlMgZGV2aWNlaWQgdXNlZCBieSBhbiBNRFMgbW91bnQuDQoNClRo
ZSBmaXggaXMgdG8gdHJhY2sgaG93IG1hbnkgcG5mcyBlbmFibGVkIGZpbGVzeXN0ZW1zIGFyZSBt
b3VudGVkIGZyb20NCnRoaXMgc2VydmVyLCBhbmQgdGhlbiB0byBwdXJnZSB0aGUgZGV2aWNlIGlk
IGNhY2hlIG9uY2UgdGhhdCBjb3VudCByZWFjaGVzDQp6ZXJvLg0KDQpSZXBvcnRlZC1ieTogSm9y
Z2UgTW9yYSA8Sm9yZ2UuTW9yYUBuZXRhcHAuY29tPg0KUmVwb3J0ZWQtYnk6IEFuZHkgQWRhbXNv
biA8YW5kcm9zQG5ldGFwcC5jb20+DQpTaWduZWQtb2ZmLWJ5OiBUcm9uZCBNeWtsZWJ1c3QgPFRy
b25kLk15a2xlYnVzdEBuZXRhcHAuY29tPg0KLS0tDQogZnMvbmZzL2NsaWVudC5jICAgICAgICAg
ICB8ICAgIDEgLQ0KIGZzL25mcy9wbmZzLmMgICAgICAgICAgICAgfCAgICA1ICsrKysrDQogaW5j
bHVkZS9saW51eC9uZnNfZnNfc2IuaCB8ICAgIDEgKw0KIDMgZmlsZXMgY2hhbmdlZCwgNiBpbnNl
cnRpb25zKCspLCAxIGRlbGV0aW9uKC0pDQoNCmRpZmYgLS1naXQgYS9mcy9uZnMvY2xpZW50LmMg
Yi9mcy9uZnMvY2xpZW50LmMNCmluZGV4IDE3YmE2YjkuLmYwMDViNWIgMTAwNjQ0DQotLS0gYS9m
cy9uZnMvY2xpZW50LmMNCisrKyBiL2ZzL25mcy9jbGllbnQuYw0KQEAgLTIwNyw3ICsyMDcsNiBA
QCBlcnJvcl8wOg0KIHN0YXRpYyB2b2lkIG5mczRfc2h1dGRvd25fc2Vzc2lvbihzdHJ1Y3QgbmZz
X2NsaWVudCAqY2xwKQ0KIHsNCiAJaWYgKG5mczRfaGFzX3Nlc3Npb24oY2xwKSkgew0KLQkJbmZz
NF9kZXZpY2VpZF9wdXJnZV9jbGllbnQoY2xwKTsNCiAJCW5mczRfZGVzdHJveV9zZXNzaW9uKGNs
cC0+Y2xfc2Vzc2lvbik7DQogCQluZnM0X2Rlc3Ryb3lfY2xpZW50aWQoY2xwKTsNCiAJfQ0KZGlm
ZiAtLWdpdCBhL2ZzL25mcy9wbmZzLmMgYi9mcy9uZnMvcG5mcy5jDQppbmRleCBiODMyM2FhLi5i
ZGY3ZTUyIDEwMDY0NA0KLS0tIGEvZnMvbmZzL3BuZnMuYw0KKysrIGIvZnMvbmZzL3BuZnMuYw0K
QEAgLTgwLDYgKzgwLDkgQEAgdW5zZXRfcG5mc19sYXlvdXRkcml2ZXIoc3RydWN0IG5mc19zZXJ2
ZXIgKm5mc3MpDQogCWlmIChuZnNzLT5wbmZzX2N1cnJfbGQpIHsNCiAJCWlmIChuZnNzLT5wbmZz
X2N1cnJfbGQtPmNsZWFyX2xheW91dGRyaXZlcikNCiAJCQluZnNzLT5wbmZzX2N1cnJfbGQtPmNs
ZWFyX2xheW91dGRyaXZlcihuZnNzKTsNCisJCS8qIERlY3JlbWVudCB0aGUgTURTIGNvdW50LiBQ
dXJnZSB0aGUgZGV2aWNlaWQgY2FjaGUgaWYgemVybyAqLw0KKwkJaWYgKGF0b21pY19kZWNfYW5k
X3Rlc3QoJm5mc3MtPm5mc19jbGllbnQtPmNsX21kc19jb3VudCkpDQorCQkJbmZzNF9kZXZpY2Vp
ZF9wdXJnZV9jbGllbnQobmZzcy0+bmZzX2NsaWVudCk7DQogCQltb2R1bGVfcHV0KG5mc3MtPnBu
ZnNfY3Vycl9sZC0+b3duZXIpOw0KIAl9DQogCW5mc3MtPnBuZnNfY3Vycl9sZCA9IE5VTEw7DQpA
QCAtMTI3LDYgKzEzMCw4IEBAIHNldF9wbmZzX2xheW91dGRyaXZlcihzdHJ1Y3QgbmZzX3NlcnZl
ciAqc2VydmVyLCBjb25zdCBzdHJ1Y3QgbmZzX2ZoICptbnRmaCwNCiAJCW1vZHVsZV9wdXQobGRf
dHlwZS0+b3duZXIpOw0KIAkJZ290byBvdXRfbm9fZHJpdmVyOw0KIAl9DQorCS8qIEJ1bXAgdGhl
IE1EUyBjb3VudCAqLw0KKwlhdG9taWNfaW5jKCZzZXJ2ZXItPm5mc19jbGllbnQtPmNsX21kc19j
b3VudCk7DQogDQogCWRwcmludGsoIiVzOiBwTkZTIG1vZHVsZSBmb3IgJXUgc2V0XG4iLCBfX2Z1
bmNfXywgaWQpOw0KIAlyZXR1cm47DQpkaWZmIC0tZ2l0IGEvaW5jbHVkZS9saW51eC9uZnNfZnNf
c2IuaCBiL2luY2x1ZGUvbGludXgvbmZzX2ZzX3NiLmgNCmluZGV4IGZiYjc4ZmIuLmY1ODMyNWEg
MTAwNjQ0DQotLS0gYS9pbmNsdWRlL2xpbnV4L25mc19mc19zYi5oDQorKysgYi9pbmNsdWRlL2xp
bnV4L25mc19mc19zYi5oDQpAQCAtMjUsNiArMjUsNyBAQCBzdHJ1Y3QgbmZzNDFfaW1wbF9pZDsN
CiAgKi8NCiBzdHJ1Y3QgbmZzX2NsaWVudCB7DQogCWF0b21pY190CQljbF9jb3VudDsNCisJYXRv
bWljX3QJCWNsX21kc19jb3VudDsNCiAJaW50CQkJY2xfY29uc19zdGF0ZTsJLyogY3VycmVudCBj
b25zdHJ1Y3Rpb24gc3RhdGUgKC12ZTogaW5pdCBlcnJvcikgKi8NCiAjZGVmaW5lIE5GU19DU19S
RUFEWQkJMAkJLyogcmVhZHkgdG8gYmUgdXNlZCAqLw0KICNkZWZpbmUgTkZTX0NTX0lOSVRJTkcJ
CTEJCS8qIGJ1c3kgaW5pdGlhbGlzaW5nICovDQotLSANCjEuNy4xMC4yDQoNCg0KLS0gDQpUcm9u
ZCBNeWtsZWJ1c3QNCkxpbnV4IE5GUyBjbGllbnQgbWFpbnRhaW5lcg0KDQpOZXRBcHANClRyb25k
Lk15a2xlYnVzdEBuZXRhcHAuY29tDQp3d3cubmV0YXBwLmNvbQ0KDQo=