[lustre-devel] [PATCH 00/20] lustre: patches landed for week of June 8 2020

[lustre-devel] [PATCH 00/20] lustre: patches landed for week of June 8 2020

[James Simmons]

Several patches from Neil tree to address bugs specific to linux
client. Backport patches from OpenSFS tree for the week of June 8
2020 and a few to support building the Lustre utilities.

Alexey Lyashkov (2):
  lustre: ptlrpc: separate number MD and refrences for bulk
  lustre: ptlrpc: fill md correctly.

Andreas Dilger (1):
  lustre: mdc: allow setting max_mod_rpcs_in_flight larger

Bobi Jam (1):
  lustre: llite: don't check mirror info for page discard

Chris Horn (1):
  lustre: ptlrpc: Clear bd_registered in ptlrpc_unregister_bulk

Di Wang (1):
  lustre: uapi: change LUSTRE_*_FL defines to enum

Lai Siyao (2):
  lustre: lmv: check stripe FID sanity
  lustre: dne: directory restripe and auto split

Mr NeilBrown (1):
  lnet: fix uninitialize var in choose_ipv4_src()

NeilBrown (5):
  lnet: fix kmalloc size in config.c
  lnet: test against LNET_STATE_RUNNING rather than LNET_STATE_SHUTDOWN
  lnet: use lnet_md_free in lnet_res_container_cleanup()
  lustre: obdclass: discard process_quota_config
  lnet: socklnd: remove comments about "darwin".

Patrick Farrell (2):
  lustre: SEL: Add flag & setstripe support
  lustre: obd: Rename OS_STATE flags to OS_STATFS

Sebastien Buisson (3):
  lustre: sec: documentation for client-side encryption
  lustre: sec: enable client side encryption
  lustre: sec: control client side encryption

Sonia Sharma (1):
  lnet: o2iblnd: 'Timed out tx' error message

 .../client_side_encryption/access_semantics.txt    | 128 +++++++++++++++++
 .../client_side_encryption/key_hierarchy.txt       | 112 +++++++++++++++
 .../lustre/client_side_encryption/modes_usage.txt  |  97 +++++++++++++
 .../lustre/client_side_encryption/threat_model.txt | 159 +++++++++++++++++++++
 fs/lustre/include/lustre_crypto.h                  | 106 ++++++++++++++
 fs/lustre/include/lustre_lmv.h                     |  20 ++-
 fs/lustre/include/lustre_net.h                     |   9 +-
 fs/lustre/include/obd_class.h                      |   4 +-
 fs/lustre/include/obd_support.h                    |   7 +
 fs/lustre/llite/Makefile                           |   1 +
 fs/lustre/llite/crypto.c                           | 132 +++++++++++++++++
 fs/lustre/llite/dir.c                              |  25 +---
 fs/lustre/llite/llite_internal.h                   |  43 ++++++
 fs/lustre/llite/llite_lib.c                        |  58 +++++++-
 fs/lustre/llite/super25.c                          |   1 +
 fs/lustre/lmv/lmv_obd.c                            |   6 +-
 fs/lustre/lov/lov_io.c                             |   9 +-
 fs/lustre/obdclass/genops.c                        |  50 +++++--
 fs/lustre/obdclass/obd_config.c                    |  10 --
 fs/lustre/ptlrpc/client.c                          |  24 +++-
 fs/lustre/ptlrpc/events.c                          |   6 +-
 fs/lustre/ptlrpc/niobuf.c                          |  40 +++---
 fs/lustre/ptlrpc/pers.c                            |  20 ++-
 fs/lustre/ptlrpc/wiretest.c                        |  29 ++--
 include/uapi/linux/lustre/lustre_idl.h             |  60 ++++++--
 include/uapi/linux/lustre/lustre_user.h            |  28 ++--
 net/lnet/klnds/o2iblnd/o2iblnd.h                   |   2 +
 net/lnet/klnds/o2iblnd/o2iblnd_cb.c                |  26 +++-
 net/lnet/klnds/socklnd/socklnd_cb.c                |   8 --
 net/lnet/lnet/api-ni.c                             |  10 +-
 net/lnet/lnet/config.c                             |   6 +-
 net/lnet/lnet/lib-socket.c                         |   2 +-
 net/lnet/lnet/peer.c                               |   6 +-
 33 files changed, 1091 insertions(+), 153 deletions(-)
 create mode 100644 Documentation/lustre/client_side_encryption/access_semantics.txt
 create mode 100644 Documentation/lustre/client_side_encryption/key_hierarchy.txt
 create mode 100644 Documentation/lustre/client_side_encryption/modes_usage.txt
 create mode 100644 Documentation/lustre/client_side_encryption/threat_model.txt
 create mode 100644 fs/lustre/include/lustre_crypto.h
 create mode 100644 fs/lustre/llite/crypto.c

-- 
1.8.3.1

[lustre-devel] [PATCH 01/20] lnet: fix kmalloc size in config.c

[James Simmons]

From: NeilBrown <neilb@suse.de>

sizeof(net->net_cpts) should be sizeof(*net->net_cpts).

For consistency, also change a sizeof(ni->ni_cpts[0]) to
sizeof(*ni->ni_cpts).

and when allocating 'array' use sizeof(*array), not
sizeof(*net->net_cpts);

Fixes: 47db2bfba415 ("lnet: add list of cpts to lnet_net.")
Reviewed-by: James Simmons <jsimmons@infradead.org>
Signed-off-by: NeilBrown <neilb@suse.de>
---
 net/lnet/lnet/config.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/net/lnet/lnet/config.c b/net/lnet/lnet/config.c
index 3fc0298..9f36c82 100644
--- a/net/lnet/lnet/config.c
+++ b/net/lnet/lnet/config.c
@@ -171,7 +171,7 @@ struct lnet_text_buf {		/* tmp struct for parsing routes */
 	}
 
 	if (!net->net_cpts) {
-		net->net_cpts = kmalloc_array(ncpts, sizeof(net->net_cpts),
+		net->net_cpts = kmalloc_array(ncpts, sizeof(*net->net_cpts),
 					      GFP_KERNEL);
 		if (!net->net_cpts)
 			return -ENOMEM;
@@ -197,7 +197,7 @@ struct lnet_text_buf {		/* tmp struct for parsing routes */
 		u32 *array = NULL, *loc;
 		u32 total_entries = j + net->net_ncpts;
 
-		array = kmalloc_array(total_entries, sizeof(*net->net_cpts),
+		array = kmalloc_array(total_entries, sizeof(*array),
 				      GFP_KERNEL);
 		if (!array) {
 			rc = -ENOMEM;
@@ -545,7 +545,7 @@ struct lnet_ni *
 	} else {
 		size_t array_size = ncpts * sizeof(ni->ni_cpts[0]);
 
-		ni->ni_cpts = kmalloc_array(ncpts, sizeof(ni->ni_cpts[0]),
+		ni->ni_cpts = kmalloc_array(ncpts, sizeof(*ni->ni_cpts),
 					    GFP_KERNEL);
 		if (!ni->ni_cpts)
 			goto failed;
-- 
1.8.3.1

[lustre-devel] [PATCH 02/20] lnet: test against LNET_STATE_RUNNING rather than LNET_STATE_SHUTDOWN

[James Simmons]

From: NeilBrown <neilb@suse.de>

The difference between LNET_STATE_STOPPING and LNET_STATE_SHUTDOWN
is of little interest.  As soon as the state isn't
LNET_STATE_RUNNING, various API calls should refuse
to act.

So change tests for SHUTDOWN to instead compare against
RUNNING.

Reviewed-by: James Simmons <jsimmons@infradead.org>
Signed-off-by: NeilBrown <neilb@suse.de>
---
 net/lnet/lnet/api-ni.c | 2 +-
 net/lnet/lnet/peer.c   | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/net/lnet/lnet/api-ni.c b/net/lnet/lnet/api-ni.c
index 62b4fa7..7c91800 100644
--- a/net/lnet/lnet/api-ni.c
+++ b/net/lnet/lnet/api-ni.c
@@ -309,7 +309,7 @@ static int lnet_discover(struct lnet_process_id id, u32 force,
 	 * changing the value dynamically, we want to set it after
 	 * updating the peers
 	 */
-	if (the_lnet.ln_state == LNET_STATE_SHUTDOWN) {
+	if (the_lnet.ln_state != LNET_STATE_RUNNING) {
 		*discovery_off = value;
 		mutex_unlock(&the_lnet.ln_api_mutex);
 		return 0;
diff --git a/net/lnet/lnet/peer.c b/net/lnet/lnet/peer.c
index 5869a20..f745394 100644
--- a/net/lnet/lnet/peer.c
+++ b/net/lnet/lnet/peer.c
@@ -810,7 +810,7 @@ struct lnet_peer_ni *
 	int rc;
 
 	rc = -ESHUTDOWN;
-	if (the_lnet.ln_state == LNET_STATE_SHUTDOWN)
+	if (the_lnet.ln_state != LNET_STATE_RUNNING)
 		goto done;
 
 	lncpt = cfs_percpt_number(the_lnet.ln_peer_tables);
@@ -1805,7 +1805,7 @@ struct lnet_peer_ni *
 	lnet_net_lock(cpt);
 
 	/* Lock has been dropped, check again for shutdown. */
-	if (the_lnet.ln_state == LNET_STATE_SHUTDOWN) {
+	if (the_lnet.ln_state != LNET_STATE_RUNNING) {
 		if (!IS_ERR(lpni))
 			lnet_peer_ni_decref_locked(lpni);
 		lpni = ERR_PTR(-ESHUTDOWN);
@@ -2816,7 +2816,7 @@ static int lnet_peer_data_present(struct lnet_peer *lp)
 	 * LNET_LOCK_EX mode is used.
 	 */
 	mutex_lock(&the_lnet.ln_api_mutex);
-	if (the_lnet.ln_state == LNET_STATE_SHUTDOWN) {
+	if (the_lnet.ln_state != LNET_STATE_RUNNING) {
 		rc = -ESHUTDOWN;
 		goto out;
 	}
-- 
1.8.3.1

[lustre-devel] [PATCH 03/20] lnet: use lnet_md_free in lnet_res_container_cleanup()

[James Simmons]

From: NeilBrown <neilb@suse.de>

When we added lnet_md_free(), we should have called it
from lnet_res_container_cleanup() - but that was missed.
Fix it now.

Reviewed-by: James Simmons <jsimmons@infradead.org>
Signed-off-by: NeilBrown <neilb@suse.de>
---
 net/lnet/lnet/api-ni.c | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/net/lnet/lnet/api-ni.c b/net/lnet/lnet/api-ni.c
index 7c91800..d6694cb 100644
--- a/net/lnet/lnet/api-ni.c
+++ b/net/lnet/lnet/api-ni.c
@@ -927,12 +927,10 @@ static void lnet_assert_wire_constants(void)
 		struct list_head *e = rec->rec_active.next;
 
 		list_del_init(e);
-		if (rec->rec_type == LNET_COOKIE_TYPE_MD) {
-			kfree(list_entry(e, struct lnet_libmd, md_list));
-
-		} else { /* NB: Active MEs should be attached on portals */
+		if (rec->rec_type == LNET_COOKIE_TYPE_MD)
+			lnet_md_free(list_entry(e, struct lnet_libmd, md_list));
+		else /* NB: Active MEs should be attached on portals */
 			LBUG();
-		}
 		count++;
 	}
 
-- 
1.8.3.1

[lustre-devel] [PATCH 04/20] lustre: obdclass: discard process_quota_config

[James Simmons]

From: NeilBrown <neilb@suse.de>

This variable is never set, so it can be discarded since it is
server side only.

Should the server side quota layer need configuration, it should
use class_modify_config().

Signed-off-by: NeilBrown <neilb@suse.de>
---
 fs/lustre/obdclass/obd_config.c | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/fs/lustre/obdclass/obd_config.c b/fs/lustre/obdclass/obd_config.c
index b8973db..5495609 100644
--- a/fs/lustre/obdclass/obd_config.c
+++ b/fs/lustre/obdclass/obd_config.c
@@ -764,11 +764,6 @@ void class_del_profiles(void)
 }
 EXPORT_SYMBOL(class_del_profiles);
 
-/* We can't call lquota_process_config directly because
- * it lives in a module that must be loaded after this one.
- */
-static int (*quota_process_config)(struct lustre_cfg *lcfg);
-
 static int process_param2_config(struct lustre_cfg *lcfg)
 {
 	char *param = lustre_cfg_string(lcfg, 1);
@@ -949,11 +944,6 @@ int class_process_config(struct lustre_cfg *lcfg)
 
 			err = 0;
 			goto out;
-		} else if ((class_match_param(lustre_cfg_string(lcfg, 1),
-					      PARAM_QUOTA, &tmp) == 0) &&
-			   quota_process_config) {
-			err = (*quota_process_config)(lcfg);
-			goto out;
 		}
 
 		break;
-- 
1.8.3.1

[lustre-devel] [PATCH 05/20] lnet: socklnd: remove comments about "darwin".

[James Simmons]

From: NeilBrown <neilb@suse.de>

As lustre only runs on Linux now, comments about
Darwin are not relevant.

Signed-off-by: NeilBrown <neilb@suse.de>
---
 net/lnet/klnds/socklnd/socklnd_cb.c | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/net/lnet/klnds/socklnd/socklnd_cb.c b/net/lnet/klnds/socklnd/socklnd_cb.c
index 2759455..623478c 100644
--- a/net/lnet/klnds/socklnd/socklnd_cb.c
+++ b/net/lnet/klnds/socklnd/socklnd_cb.c
@@ -701,10 +701,6 @@ struct ksock_conn *
 					      KSOCK_MSG_NOOP,
 	       tx->tx_nob, tx->tx_niov, tx->tx_nkiov);
 
-	/*
-	 * FIXME: SOCK_WMEM_QUEUED and SOCK_ERROR could block in __DARWIN8__
-	 * but they're used inside spinlocks a lot.
-	 */
 	bufnob = conn->ksnc_sock->sk->sk_wmem_queued;
 	spin_lock_bh(&sched->kss_lock);
 
@@ -2203,10 +2199,6 @@ void ksocknal_write_callback(struct ksock_conn *conn)
 		/* Don't need the {get,put}connsock dance to deref ksnc_sock */
 		LASSERT(!conn->ksnc_closing);
 
-		/*
-		 * SOCK_ERROR will reset error code of socket in
-		 * some platform (like Darwin8.x)
-		 */
 		error = conn->ksnc_sock->sk->sk_err;
 		if (error) {
 			ksocknal_conn_addref(conn);
-- 
1.8.3.1

[lustre-devel] [PATCH 06/20] lustre: uapi: change LUSTRE_*_FL defines to enum

[James Simmons]

From: Di Wang <di.wang@intel.com>

The preferred style in the case of LUSTRE_*_FL is to use enum.

Fixes: d344c54f4a90c ("lustre: llite: handle ORPHAN/DEAD directories")
WC-bug-id: https://jira.whamcloud.com/browse/LU-7579
Lustre-commit: 098fb363c39 ("LU-7579 osd: move ORPHAN/DEAD flag to OSD")
Signed-off-by: Di Wang <di.wang@intel.com>
Reviewed-on: http://review.whamcloud.com/18024
Reviewed-by: John L. Hammond <jhammond@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 include/uapi/linux/lustre/lustre_idl.h | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/include/uapi/linux/lustre/lustre_idl.h b/include/uapi/linux/lustre/lustre_idl.h
index bb3f5f1..800154d 100644
--- a/include/uapi/linux/lustre/lustre_idl.h
+++ b/include/uapi/linux/lustre/lustre_idl.h
@@ -1537,19 +1537,23 @@ enum {
 #define MDS_STATUS_CONN 1
 #define MDS_STATUS_LOV 2
 
-/* these should be identical to their EXT4_*_FL counterparts, they are
- * redefined here only to avoid dragging in fs/ext4/ext4.h
- */
-#define LUSTRE_SYNC_FL		0x00000008 /* Synchronous updates */
-#define LUSTRE_IMMUTABLE_FL	0x00000010 /* Immutable file */
-#define LUSTRE_APPEND_FL	0x00000020 /* writes to file may only append */
-#define LUSTRE_NODUMP_FL	0x00000040 /* do not dump file */
-#define LUSTRE_NOATIME_FL	0x00000080 /* do not update atime */
-#define LUSTRE_INDEX_FL		0x00001000 /* hash-indexed directory */
-#define LUSTRE_DIRSYNC_FL	0x00010000 /* dirsync behaviour (dir only) */
-#define LUSTRE_TOPDIR_FL	0x00020000 /* Top of directory hierarchies*/
-#define LUSTRE_INLINE_DATA_FL	0x10000000 /* Inode has inline data. */
-#define LUSTRE_PROJINHERIT_FL	0x20000000 /* Create with parents projid */
+enum {
+	/* these should be identical to their EXT4_*_FL counterparts, they are
+	 * redefined here only to avoid dragging in fs/ext4/ext4.h
+	 */
+	LUSTRE_SYNC_FL		= 0x00000008, /* Synchronous updates */
+	LUSTRE_IMMUTABLE_FL	= 0x00000010, /* Immutable file */
+	LUSTRE_APPEND_FL	= 0x00000020, /* writes to file may only
+					       * append
+					       */
+	LUSTRE_NODUMP_FL	= 0x00000040, /* do not dump file */
+	LUSTRE_NOATIME_FL	= 0x00000080, /* do not update atime */
+	LUSTRE_INDEX_FL		= 0x00001000, /* hash-indexed directory */
+	LUSTRE_DIRSYNC_FL	= 0x00010000, /* dirsync behaviour (dir only) */
+	LUSTRE_TOPDIR_FL	= 0x00020000, /* Top of directory hierarchies*/
+	LUSTRE_INLINE_DATA_FL	= 0x10000000, /* Inode has inline data. */
+	LUSTRE_PROJINHERIT_FL	= 0x20000000, /* Create with parents projid */
+};
 
 /* 64 possible states */
 enum md_transient_state {
-- 
1.8.3.1

[lustre-devel] [PATCH 07/20] lustre: SEL: Add flag & setstripe support

[James Simmons]

From: Patrick Farrell <paf@cray.com>

The self-extending layouts feature adds a new layout flag
and also uses the stripe size field differently.

This patch implements this basic functionality, to be used
in subsequent patches.

Cray-bug-id: LUS-2528
WC-bug-id: https://jira.whamcloud.com/browse/LU-10070
Lustre-commit: fed241911f61b ("LU-10070 lod: SEL: Add flag & setstripe support")
Signed-off-by: Patrick Farrell <paf@cray.com>
Reviewed-on: https://review.whamcloud.com/33782
Reviewed-by: Alexey Lyashkov <c17817@cray.com>
Reviewed-by: Patrick Farrell <pfarrell@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 fs/lustre/ptlrpc/wiretest.c             | 1 +
 include/uapi/linux/lustre/lustre_user.h | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/fs/lustre/ptlrpc/wiretest.c b/fs/lustre/ptlrpc/wiretest.c
index dc1d0af..719d73b 100644
--- a/fs/lustre/ptlrpc/wiretest.c
+++ b/fs/lustre/ptlrpc/wiretest.c
@@ -1659,6 +1659,7 @@ void lustre_assert_wire_constants(void)
 	BUILD_BUG_ON(LCME_FL_OFFLINE != 0x00000008);
 	BUILD_BUG_ON(LCME_FL_INIT != 0x00000010);
 	BUILD_BUG_ON(LCME_FL_NOSYNC != 0x00000020);
+	BUILD_BUG_ON(LCME_FL_EXTENSION != 0x00000040);
 	BUILD_BUG_ON(LCME_FL_NEG != 0x80000000);
 
 	/* Checks for struct lov_comp_md_v1 */
diff --git a/include/uapi/linux/lustre/lustre_user.h b/include/uapi/linux/lustre/lustre_user.h
index b255f14..004d373 100644
--- a/include/uapi/linux/lustre/lustre_user.h
+++ b/include/uapi/linux/lustre/lustre_user.h
@@ -548,17 +548,17 @@ enum lov_comp_md_entry_flags {
 	LCME_FL_OFFLINE		= 0x00000008,	/* Not used */
 	LCME_FL_INIT		= 0x00000010,	/* instantiated */
 	LCME_FL_NOSYNC		= 0x00000020,	/* FLR: no sync for the mirror */
+	LCME_FL_EXTENSION	= 0x00000040,	/* extension comp, never init */
 	LCME_FL_NEG		= 0x80000000,	/* used to indicate a negative
 						 * flag, won't be stored on disk
 						 */
 };
 
 #define LCME_KNOWN_FLAGS	(LCME_FL_NEG | LCME_FL_INIT | LCME_FL_STALE | \
-				 LCME_FL_PREF_RW | LCME_FL_NOSYNC)
-/* The flags can be set by users at mirror creation time. */
-#define LCME_USER_FLAGS		(LCME_FL_PREF_RW)
+				 LCME_FL_PREF_RW | LCME_FL_NOSYNC | \
+				 LCME_FL_EXTENSION)
 
-/* The flags are for mirrors */
+/* The mirror flags sent by client */
 #define LCME_MIRROR_FLAGS	(LCME_FL_NOSYNC)
 
 /* the highest bit in obdo::o_layout_version is used to mark if the file is
-- 
1.8.3.1

[lustre-devel] [PATCH 08/20] lustre: lmv: check stripe FID sanity

[James Simmons]

From: Lai Siyao <lai.siyao@whamcloud.com>

Striped directory layout may be broken, if some stripe FID is insane,
return -ENODEV.

WC-bug-id: https://jira.whamcloud.com/browse/LU-13437
Lustre-commit: 698a496aac51e ("LU-13437 lmv: check stripe FID sanity")
Signed-off-by: Lai Siyao <lai.siyao@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/38560
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Hongchao Zhang <hongchao@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 fs/lustre/lmv/lmv_obd.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/fs/lustre/lmv/lmv_obd.c b/fs/lustre/lmv/lmv_obd.c
index 4583cea..203a17b 100644
--- a/fs/lustre/lmv/lmv_obd.c
+++ b/fs/lustre/lmv/lmv_obd.c
@@ -1539,6 +1539,10 @@ static struct lu_tgt_desc *lmv_locate_tgt_rr(struct lmv_obd *lmv, u32 *mdt)
 			return ERR_CAST(oinfo);
 	}
 
+	/* check stripe FID is sane */
+	if (!fid_is_sane(&oinfo->lmo_fid))
+		return ERR_PTR(-ENODEV);
+
 	*fid = oinfo->lmo_fid;
 	*mds = oinfo->lmo_mds;
 	tgt = lmv_tgt(lmv, oinfo->lmo_mds);
-- 
1.8.3.1

[lustre-devel] [PATCH 09/20] lustre: ptlrpc: Clear bd_registered in ptlrpc_unregister_bulk

[James Simmons]

From: Chris Horn <hornc@cray.com>

The patch for LU-12816 https://review.whamcloud.com/36309 has us
clearing the bd_registered flag in ptl_send_rpc(). This flag is set
in ptlrpc_register_bulk(), so it makes sense for us to clear it in
ptlrpc_unregister_bulk(). When we're cleaning up in ptl_send_rpc()
we can be sure the flag is cleared with the call to
ptlrpc_unregister_bulk().

This commit also adds a test case for the LU-12816 bug.

Fixes: 1010595a9bb6 ("lustre: ptlrpc: ptlrpc_register_bulk LBUG on ENOMEM")
WC-bug-id: https://jira.whamcloud.com/browse/LU-13509
Lustre-commit: 15057a17ca1e2 ("LU-13509 ptlrpc: Clear bd_registered in ptlrpc_unregister_bulk")
Signed-off-by: Chris Horn <hornc@cray.com>
Reviewed-on: https://review.whamcloud.com/38457
Reviewed-by: Shaun Tancheff <shaun.tancheff@hpe.com>
Reviewed-by: Wang Shilong <wshilong@ddn.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 fs/lustre/include/obd_support.h |  1 +
 fs/lustre/ptlrpc/niobuf.c       | 18 +++++++++++++-----
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/fs/lustre/include/obd_support.h b/fs/lustre/include/obd_support.h
index b706a20..d1d21e0 100644
--- a/fs/lustre/include/obd_support.h
+++ b/fs/lustre/include/obd_support.h
@@ -362,6 +362,7 @@
 #define OBD_FAIL_PTLRPC_LONG_REQ_UNLINK			0x51b
 #define OBD_FAIL_PTLRPC_LONG_BOTH_UNLINK		0x51c
 #define OBD_FAIL_PTLRPC_BULK_ATTACH			0x521
+#define OBD_FAIL_PTLRPC_BULK_REPLY_ATTACH		0x522
 #define OBD_FAIL_PTLRPC_ROUND_XID			0x530
 #define OBD_FAIL_PTLRPC_CONNECT_RACE			0x531
 
diff --git a/fs/lustre/ptlrpc/niobuf.c b/fs/lustre/ptlrpc/niobuf.c
index d62629a..331a0c8 100644
--- a/fs/lustre/ptlrpc/niobuf.c
+++ b/fs/lustre/ptlrpc/niobuf.c
@@ -250,6 +250,9 @@ int ptlrpc_unregister_bulk(struct ptlrpc_request *req, int async)
 
 	LASSERT(!in_interrupt());     /* might sleep */
 
+	if (desc)
+		desc->bd_registered = 0;
+
 	/* Let's setup deadline for reply unlink. */
 	if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_BULK_UNLINK) &&
 	    async && req->rq_bulk_deadline == 0 && cfs_fail_val == 0)
@@ -614,9 +617,16 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
 			request->rq_repmsg = NULL;
 		}
 
-		reply_me = LNetMEAttach(request->rq_reply_portal,
-					connection->c_peer, request->rq_xid, 0,
-					LNET_UNLINK, LNET_INS_AFTER);
+		if (request->rq_bulk &&
+		    OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_BULK_REPLY_ATTACH)) {
+			reply_me = ERR_PTR(-ENOMEM);
+		} else {
+			reply_me = LNetMEAttach(request->rq_reply_portal,
+						connection->c_peer,
+						request->rq_xid, 0,
+						LNET_UNLINK, LNET_INS_AFTER);
+		}
+
 		if (IS_ERR(reply_me)) {
 			rc = PTR_ERR(reply_me);
 			CERROR("LNetMEAttach failed: %d\n", rc);
@@ -724,8 +734,6 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
 	 * the chance to have long unlink to sluggish net is smaller here.
 	 */
 	ptlrpc_unregister_bulk(request, 0);
-	if (request->rq_bulk)
-		request->rq_bulk->bd_registered = 0;
 out:
 	if (rc == -ENOMEM) {
 		/*
-- 
1.8.3.1

[lustre-devel] [PATCH 10/20] lustre: dne: directory restripe and auto split

[James Simmons]

From: Lai Siyao <lai.siyao@whamcloud.com>

A specific restriper thread is created for each MDT, it does three
tasks in a loop:
1. If there is directory whose total sub-files exceeds threshold
   (50000 by default, can be changed "lctl set_param
   mdt.*.dir_split_count=N"), split this directory by adding new
   stripes (4 stripes by default, which can be adjusted by
   "lctl set_param mdt.*.dir_split_delta=N").
2. If a directory stripe LMV is marked 'MIGRATION', migrate sub file
   from current offset, and update offset to next file.
3. If a directory master LMV is marked 'RESTRIPING', check whether
   all stripe LMV 'MIGRATION' flag is cleared, if so, clear
   'RESTRIPING' flag and update directory LMV.

In last patch, the first part of manual directory stripe is
implemented, and in this patch, sub file migrations and dir layout
update is done. Directory auto-split is done in similar way, except
that the first step is done by this thread too.

Directory auto-split can be enabled/disabled by "lctl set_param
mdt.*.enable_dir_auto_split=[0|1]", it's turned on by default.

Auto split is triggered at the end of getattr(): since now the attr
contains dirent count, check whether it exceeds threshold, if so,
add this directory into mdr_auto_split list and wake up the dir
restriper thread.

Restripe migration is also triggered in getattr(): if the object is
directory stripe, and LMV 'MIGRATION' flag set, add this object into
mdr_restripe_migrate list and wake up the dir restriper thread.

Directory layout update is similar: if current directory is striped,
and LNV 'RESTRIPING' flag is set, add this directory into
mdr_restripe_update list and wake up restriper thread.

By default restripe migrate dirent only, and leave inode unchanged, it
can be adjusted by "lctl set_param mdt.*.dir_restripe_nsonly=[0|1]".

Currently DoM file inode migration is not supported, migrate dirent
only for such files to avoid leaving dir migration/restripe
unfinished.

WC-bug-id: https://jira.whamcloud.com/browse/LU-11025
Lustre-commit: a336d7c7c1cd6 ("LU-11025 dne: directory restripe and auto split")
Signed-off-by: Lai Siyao <lai.siyao@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/37284
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Hongchao Zhang <hongchao@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 fs/lustre/include/lustre_lmv.h         | 20 +++++++++++++++-----
 fs/lustre/llite/dir.c                  | 25 ++-----------------------
 include/uapi/linux/lustre/lustre_idl.h | 18 ++++++++++++++++++
 3 files changed, 35 insertions(+), 28 deletions(-)

diff --git a/fs/lustre/include/lustre_lmv.h b/fs/lustre/include/lustre_lmv.h
index 0175816..afa4d60 100644
--- a/fs/lustre/include/lustre_lmv.h
+++ b/fs/lustre/include/lustre_lmv.h
@@ -452,32 +452,42 @@ static inline bool lmv_is_sane2(const struct lmv_mds_md_v1 *lmv)
 
 static inline bool lmv_is_splitting(const struct lmv_mds_md_v1 *lmv)
 {
-	LASSERT(lmv_is_sane2(lmv));
+	if (!lmv_is_sane2(lmv))
+		return false;
+
 	return lmv_hash_is_splitting(cpu_to_le32(lmv->lmv_hash_type));
 }
 
 static inline bool lmv_is_merging(const struct lmv_mds_md_v1 *lmv)
 {
-	LASSERT(lmv_is_sane2(lmv));
+	if (!lmv_is_sane2(lmv))
+		return false;
+
 	return lmv_hash_is_merging(cpu_to_le32(lmv->lmv_hash_type));
 }
 
 static inline bool lmv_is_migrating(const struct lmv_mds_md_v1 *lmv)
 {
-	LASSERT(lmv_is_sane(lmv));
+	if (!lmv_is_sane(lmv))
+		return false;
+
 	return lmv_hash_is_migrating(cpu_to_le32(lmv->lmv_hash_type));
 }
 
 static inline bool lmv_is_restriping(const struct lmv_mds_md_v1 *lmv)
 {
-	LASSERT(lmv_is_sane2(lmv));
+	if (!lmv_is_sane2(lmv))
+		return false;
+
 	return lmv_hash_is_splitting(cpu_to_le32(lmv->lmv_hash_type)) ||
 	       lmv_hash_is_merging(cpu_to_le32(lmv->lmv_hash_type));
 }
 
 static inline bool lmv_is_layout_changing(const struct lmv_mds_md_v1 *lmv)
 {
-	LASSERT(lmv_is_sane2(lmv));
+	if (!lmv_is_sane2(lmv))
+		return false;
+
 	return lmv_hash_is_splitting(cpu_to_le32(lmv->lmv_hash_type)) ||
 	       lmv_hash_is_merging(cpu_to_le32(lmv->lmv_hash_type)) ||
 	       lmv_hash_is_migrating(cpu_to_le32(lmv->lmv_hash_type));
diff --git a/fs/lustre/llite/dir.c b/fs/lustre/llite/dir.c
index 7fd52fe..0ffe134 100644
--- a/fs/lustre/llite/dir.c
+++ b/fs/lustre/llite/dir.c
@@ -48,6 +48,7 @@
 
 #include <obd_support.h>
 #include <obd_class.h>
+#include <uapi/linux/lustre/lustre_idl.h>
 #include <uapi/linux/lustre/lustre_ioctl.h>
 #include <lustre_lib.h>
 #include <lustre_dlm.h>
@@ -174,28 +175,6 @@ void ll_release_page(struct inode *inode, struct page *page, bool remove)
 	put_page(page);
 }
 
-/**
- * return IF_* type for given lu_dirent entry.
- * IF_* flag shld be converted to particular OS file type in
- * platform llite module.
- */
-static u16 ll_dirent_type_get(struct lu_dirent *ent)
-{
-	u16 type = 0;
-	struct luda_type *lt;
-	int len = 0;
-
-	if (le32_to_cpu(ent->lde_attrs) & LUDA_TYPE) {
-		const unsigned int align = sizeof(struct luda_type) - 1;
-
-		len = le16_to_cpu(ent->lde_namelen);
-		len = (len + align) & ~align;
-		lt = (void *)ent->lde_name + len;
-		type = IFTODT(le16_to_cpu(lt->lt_type));
-	}
-	return type;
-}
-
 int ll_dir_read(struct inode *inode, u64 *ppos, struct md_op_data *op_data,
 		struct dir_context *ctx)
 {
@@ -246,7 +225,7 @@ int ll_dir_read(struct inode *inode, u64 *ppos, struct md_op_data *op_data,
 				lhash = hash;
 			fid_le_to_cpu(&fid, &ent->lde_fid);
 			ino = cl_fid_build_ino(&fid, is_api32);
-			type = ll_dirent_type_get(ent);
+			type = IFTODT(lu_dirent_type_get(ent));
 			ctx->pos = lhash;
 			/* For 'll_nfs_get_name_filldir()', it will try
 			 * to access the 'ent' through its 'lde_name',
diff --git a/include/uapi/linux/lustre/lustre_idl.h b/include/uapi/linux/lustre/lustre_idl.h
index 800154d..d0b43c8 100644
--- a/include/uapi/linux/lustre/lustre_idl.h
+++ b/include/uapi/linux/lustre/lustre_idl.h
@@ -492,6 +492,24 @@ static inline __kernel_size_t lu_dirent_calc_size(size_t namelen, __u16 attr)
 	return (size + 7) & ~7;
 }
 
+static inline __u16 lu_dirent_type_get(struct lu_dirent *ent)
+{
+	__u16 type = 0;
+	struct luda_type *lt;
+	int len = 0;
+
+	if (__le32_to_cpu(ent->lde_attrs) & LUDA_TYPE) {
+		const unsigned int align = sizeof(struct luda_type) - 1;
+
+		len = __le16_to_cpu(ent->lde_namelen);
+		len = (len + align) & ~align;
+		lt = (void *)ent->lde_name + len;
+		type = __le16_to_cpu(lt->lt_type);
+	}
+
+	return type;
+}
+
 #define MDS_DIR_END_OFF 0xfffffffffffffffeULL
 
 /**
-- 
1.8.3.1

[lustre-devel] [PATCH 11/20] lustre: sec: documentation for client-side encryption

[James Simmons]

From: Sebastien Buisson <sbuisson@ddn.com>

Add several documents about client-side encryption under
Documentation/client_side_encryption:
- threat_model.txt is the description of the threat model for Lustre
  client-side encryption;
- key_hierarchy.txt is the description of the key hierarchy for Lustre
  client-side encryption;
- modes_usage.txt is the description of the encryption modes and usage
  for Lustre client-side encryption;
- access_semantics.txt is the description of the access semantics for
  Lustre client-side encryption.

As we rely on kernel's fscrypt library for this feature, fscrypt's
concepts are largely valid. These documents are inspired by fscrypt
documentation in the Linux kernel tree, see
Documentation/filesystems/fscrypt.rst

WC-bug-id: https://jira.whamcloud.com/browse/LU-12275
Lustre-commit: f2bf564e691ea ("LU-12275 sec: documentation for client-side encryption")
Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Reviewed-on: https://review.whamcloud.com/38759
Reviewed-by: John L. Hammond <jhammond@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 .../client_side_encryption/access_semantics.txt    | 128 +++++++++++++++++
 .../client_side_encryption/key_hierarchy.txt       | 112 +++++++++++++++
 .../lustre/client_side_encryption/modes_usage.txt  |  97 +++++++++++++
 .../lustre/client_side_encryption/threat_model.txt | 159 +++++++++++++++++++++
 4 files changed, 496 insertions(+)
 create mode 100644 Documentation/lustre/client_side_encryption/access_semantics.txt
 create mode 100644 Documentation/lustre/client_side_encryption/key_hierarchy.txt
 create mode 100644 Documentation/lustre/client_side_encryption/modes_usage.txt
 create mode 100644 Documentation/lustre/client_side_encryption/threat_model.txt

diff --git a/Documentation/lustre/client_side_encryption/access_semantics.txt b/Documentation/lustre/client_side_encryption/access_semantics.txt
new file mode 100644
index 0000000..fe2c28d
--- /dev/null
+++ b/Documentation/lustre/client_side_encryption/access_semantics.txt
@@ -0,0 +1,128 @@
+===============================================
+Lustre client-level encryption access semantics
+===============================================
+
+Lustre client-level encryption relies on kernel's fscrypt, and more
+precisely on v2 encryption policies.
+fscrypt is a library which filesystems can hook into to support
+transparent encryption of files and directories.
+
+As a consequence, the access semantics described here, extracted from
+fscrypt's doc, apply to Lustre client-level encryption.
+
+Ref:
+https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/fscrypt.rst
+
+Access
+======
+
+Only Lustre clients need access to encryption master keys. Keys are
+added to the filesystem-level encryption keyring on the Lustre client.
+
+With the key
+------------
+
+With the encryption key, encrypted regular files, directories, and
+symlinks behave very similarly to their unencrypted counterparts ---
+after all, the encryption is intended to be transparent.  However,
+astute users may notice some differences in behavior:
+
+- Unencrypted files, or files encrypted with a different encryption
+  policy (i.e. different key, modes, or flags), cannot be renamed or
+  linked into an encrypted directory; see `Encryption policy
+  enforcement`_.  Attempts to do so will fail with EXDEV.  However,
+  encrypted files can be renamed within an encrypted directory, or
+  into an unencrypted directory.
+
+  Note: "moving" an unencrypted file into an encrypted directory, e.g.
+  with the `mv` program, is implemented in userspace by a copy
+  followed by a delete.  Be aware that the original unencrypted data
+  may remain recoverable from free space on the disk; prefer to keep
+  all files encrypted from the very beginning.  The `shred` program
+  may be used to overwrite the source files but isn't guaranteed to be
+  effective on all filesystems and storage devices.
+
+- Direct I/O is not supported on encrypted files.  Attempts to use
+  direct I/O on such files will fall back to buffered I/O.
+
+- The fallocate operations FALLOC_FL_COLLAPSE_RANGE,
+  FALLOC_FL_INSERT_RANGE, and FALLOC_FL_ZERO_RANGE are not supported
+  on encrypted files and will fail with EOPNOTSUPP.
+
+- DAX (Direct Access) is not supported on encrypted files.
+
+- The st_size of an encrypted symlink will not necessarily give the
+  length of the symlink target as required by POSIX.  It will actually
+  give the length of the ciphertext, which will be slightly longer
+  than the plaintext due to NUL-padding and an extra 2-byte overhead.
+
+- The maximum length of an encrypted symlink is 2 bytes shorter than
+  the maximum length of an unencrypted symlink.  For example, on an
+  EXT4 filesystem with a 4K block size, unencrypted symlinks can be up
+  to 4095 bytes long, while encrypted symlinks can only be up to 4093
+  bytes long (both lengths excluding the terminating null).
+
+Note that mmap *is* supported.  This is possible because the pagecache
+for an encrypted file contains the plaintext, not the ciphertext.
+
+Without the key
+---------------
+
+Some filesystem operations may be performed on encrypted regular
+files, directories, and symlinks even before their encryption key has
+been added, or after their encryption key has been removed:
+
+- File metadata may be read, e.g. using stat().
+
+- Directories may be listed, in which case the filenames will be
+  listed in an encoded form derived from their ciphertext.  The
+  algorithm is subject to change, but it is guaranteed that the
+  presented filenames will be no longer than NAME_MAX bytes, will not
+  contain the ``/`` or ``\0`` characters, and will uniquely identify
+  directory entries.
+
+  The ``.`` and ``..`` directory entries are special.  They are always
+  present and are not encrypted or encoded.
+
+- Files may be deleted.  That is, nondirectory files may be deleted
+  with unlink() as usual, and empty directories may be deleted with
+  rmdir() as usual.  Therefore, ``rm`` and ``rm -r`` will work as
+  expected.
+
+- Symlink targets may be read and followed, but they will be presented
+  in encrypted form, similar to filenames in directories.  Hence, they
+  are unlikely to point to anywhere useful.
+
+Without the key, regular files cannot be opened or truncated.
+Attempts to do so will fail with ENOKEY.  This implies that any
+regular file operations that require a file descriptor, such as
+read(), write(), mmap(), fallocate(), and ioctl(), are also forbidden.
+
+Also without the key, files of any type (including directories) cannot
+be created or linked into an encrypted directory, nor can a name in an
+encrypted directory be the source or target of a rename, nor can an
+O_TMPFILE temporary file be created in an encrypted directory.  All
+such operations will fail with ENOKEY.
+
+It is not currently possible to backup and restore encrypted files
+without the encryption key.  This would require special APIs which
+have not yet been implemented.
+
+Encryption policy enforcement
+=============================
+
+After an encryption policy has been set on a directory, all regular
+files, directories, and symbolic links created in that directory
+(recursively) will inherit that encryption policy.  Special files ---
+that is, named pipes, device nodes, and UNIX domain sockets --- will
+not be encrypted.
+
+Except for those special files, it is forbidden to have unencrypted
+files, or files encrypted with a different encryption policy, in an
+encrypted directory tree.  Attempts to link or rename such a file into
+an encrypted directory will fail with EXDEV.  This is also enforced
+during ->lookup() to provide limited protection against offline
+attacks that try to disable or downgrade encryption in known locations
+where applications may later write sensitive data.  It is recommended
+that systems implementing a form of "verified boot" take advantage of
+this by validating all top-level encryption policies prior to access.
diff --git a/Documentation/lustre/client_side_encryption/key_hierarchy.txt b/Documentation/lustre/client_side_encryption/key_hierarchy.txt
new file mode 100644
index 0000000..d96e189
--- /dev/null
+++ b/Documentation/lustre/client_side_encryption/key_hierarchy.txt
@@ -0,0 +1,112 @@
+============================================
+Lustre client-level encryption key hierarchy
+============================================
+
+Lustre client-level encryption relies on kernel's fscrypt, and more
+precisely on v2 encryption policies.
+fscrypt is a library which filesystems can hook into to support
+transparent encryption of files and directories.
+
+As a consequence, the following key hierarchy description, extracted
+from fscrypt's, is directly applicable to Lustre client-level encryption.
+
+Ref:
+https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/fscrypt.rst
+
+Master Keys
+-----------
+
+Each encrypted directory tree is protected by a *master key*.  Master
+keys can be up to 64 bytes long, and must be at least as long as the
+greater of the key length needed by the contents and filenames
+encryption modes being used.  For example, if AES-256-XTS is used for
+contents encryption, the master key must be 64 bytes (512 bits).  Note
+that the XTS mode is defined to require a key twice as long as that
+required by the underlying block cipher.
+
+To "unlock" an encrypted directory tree, userspace must provide the
+appropriate master key.  There can be any number of master keys, each
+of which protects any number of directory trees on any number of
+filesystems.
+
+Master keys should be pseudorandom, i.e. indistinguishable from random
+bytestrings of the same length.  This implies that users **must not**
+directly use a password as a master key, zero-pad a shorter key, or
+repeat a shorter key.  Instead, users should generate master keys
+either using a cryptographically secure random number generator, or by
+using a KDF (Key Derivation Function).  Note that whenever a KDF is
+used to "stretch" a lower-entropy secret such as a passphrase, it is
+critical that a KDF designed for this purpose be used, such as scrypt,
+PBKDF2, or Argon2.
+
+Key derivation function
+-----------------------
+
+With one exception, fscrypt never uses the master key(s) for
+encryption directly.  Instead, they are only used as input to a KDF
+(Key Derivation Function) to derive the actual keys.
+
+For v2 encryption policies, the KDF is HKDF-SHA512. The master key is
+passed as the "input keying material", no salt is used, and a distinct
+"application-specific information string" is used for each distinct
+key to be derived.  For example, when a per-file encryption key is
+derived, the application-specific information string is the file's
+nonce prefixed with "fscrypt\0" and a context byte.  Different context
+bytes are used for other types of derived keys.
+
+HKDF-SHA512 is preferred KDF because HKDF is more flexible, is
+nonreversible, and evenly distributes entropy from the master key.
+HKDF is also standardized and widely used by other software.
+
+Per-file keys
+-------------
+
+Since each master key can protect many files, it is necessary to
+"tweak" the encryption of each file so that the same plaintext in two
+files doesn't map to the same ciphertext, or vice versa.  In most
+cases, fscrypt does this by deriving per-file keys.  When a new
+encrypted inode (regular file, directory, or symlink) is created,
+fscrypt randomly generates a 16-byte nonce and stores it in the
+inode's encryption xattr.  Then, it uses a KDF (as described in `Key
+derivation function`_) to derive the file's key from the master key
+and nonce.
+
+Key derivation was chosen over key wrapping because wrapped keys would
+require larger xattrs which would be less likely to fit in-line in the
+filesystem's inode table, and there didn't appear to be any
+significant advantages to key wrapping.  In particular, currently
+there is no requirement to support unlocking a file with multiple
+alternative master keys or to support rotating master keys.  Instead,
+the master keys may be wrapped in userspace, e.g. as is done by the
+`fscrypt <https://github.com/google/fscrypt>`_ tool.
+
+Including the inode number in the IVs was considered.  However, it was
+rejected as it would have prevented ext4 filesystems from being
+resized, and by itself still wouldn't have been sufficient to prevent
+the same key from being directly reused for both XTS and CTS-CBC.
+
+DIRECT_KEY and per-mode keys
+----------------------------
+
+The Adiantum encryption mode is suitable for both contents and
+filenames encryption, and it accepts long IVs --- long enough to hold
+both an 8-byte logical block number and a 16-byte per-file nonce.
+Also, the overhead of each Adiantum key is greater than that of an
+AES-256-XTS key.
+
+Therefore, to improve performance and save memory, for Adiantum a
+"direct key" configuration is supported.  When the user has enabled
+this by setting FSCRYPT_POLICY_FLAG_DIRECT_KEY in the fscrypt policy,
+per-file keys are not used.  Instead, whenever any data (contents or
+filenames) is encrypted, the file's 16-byte nonce is included in the
+IV.  Moreover, for v2 encryption policies, the encryption is done with
+a per-mode key derived using the KDF.  Users may use the same master
+key for other v2 encryption policies.
+
+Key identifiers
+---------------
+
+For master keys used for v2 encryption policies, a unique 16-byte "key
+identifier" is also derived using the KDF.  This value is stored in
+the clear, since it is needed to reliably identify the key itself.
+
diff --git a/Documentation/lustre/client_side_encryption/modes_usage.txt b/Documentation/lustre/client_side_encryption/modes_usage.txt
new file mode 100644
index 0000000..4ad8cf5
--- /dev/null
+++ b/Documentation/lustre/client_side_encryption/modes_usage.txt
@@ -0,0 +1,97 @@
+==============================================
+Lustre client-level encryption modes and usage
+==============================================
+
+Lustre client-level encryption relies on kernel's fscrypt, and more
+precisely on v2 encryption policies.
+fscrypt is a library which filesystems can hook into to support
+transparent encryption of files and directories.
+
+As a consequence, the encryption modes and usage described here,
+extracted from fscrypt's doc, apply to Lustre client-level encryption.
+
+Ref:
+https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/fscrypt.rst
+
+Encryption modes
+----------------
+
+fscrypt allows one encryption mode to be specified for file contents
+and one encryption mode to be specified for filenames.  Different
+directory trees are permitted to use different encryption modes.
+Currently, the following pairs of encryption modes are supported:
+
+- AES-256-XTS for contents and AES-256-CTS-CBC for filenames
+- AES-128-CBC for contents and AES-128-CTS-CBC for filenames
+- Adiantum for both contents and filenames
+
+If unsure, you should use the (AES-256-XTS, AES-256-CTS-CBC) pair.
+
+AES-128-CBC was added only for low-powered embedded devices with
+crypto accelerators such as CAAM or CESA that do not support XTS.
+
+Adiantum is a (primarily) stream cipher-based mode that is fast even
+on CPUs without dedicated crypto instructions.  It's also a true
+wide-block mode, unlike XTS.  It can also eliminate the need to derive
+per-file keys.  However, it depends on the security of two primitives,
+XChaCha12 and AES-256, rather than just one.  See the paper
+"Adiantum: length-preserving encryption for entry-level processors"
+(https://eprint.iacr.org/2018/720.pdf) for more details.  To use
+Adiantum, CONFIG_CRYPTO_ADIANTUM must be enabled.  Also, fast
+implementations of ChaCha and NHPoly1305 should be enabled, e.g.
+CONFIG_CRYPTO_CHACHA20_NEON and CONFIG_CRYPTO_NHPOLY1305_NEON for ARM.
+
+Contents encryption
+-------------------
+
+For file contents, each filesystem block is encrypted independently.
+Currently, only the case where the filesystem block size is equal to
+the system's page size (usually 4096 bytes) is supported.
+
+Each block's IV is set to the logical block number within the file as
+a little endian number, except that:
+
+- With CBC mode encryption, ESSIV is also used.  Specifically, each IV
+  is encrypted with AES-256 where the AES-256 key is the SHA-256 hash
+  of the file's data encryption key.
+
+- In the "direct key" configuration (FSCRYPT_POLICY_FLAG_DIRECT_KEY
+  set in the fscrypt_policy), the file's nonce is also appended to the
+  IV.  Currently this is only allowed with the Adiantum encryption
+  mode.
+
+Filenames encryption
+--------------------
+
+For filenames, each full filename is encrypted at once.  Because of
+the requirements to retain support for efficient directory lookups and
+filenames of up to 255 bytes, the same IV is used for every filename
+in a directory.
+
+However, each encrypted directory still uses a unique key; or
+alternatively (for the "direct key" configuration) has the file's
+nonce included in the IVs.  Thus, IV reuse is limited to within a
+single directory.
+
+With CTS-CBC, the IV reuse means that when the plaintext filenames
+share a common prefix at least as long as the cipher block size (16
+bytes for AES), the corresponding encrypted filenames will also share
+a common prefix.  This is undesirable.  Adiantum does not have this
+weakness, as it is a wide-block encryption mode.
+
+All supported filenames encryption modes accept any plaintext length
+>= 16 bytes; cipher block alignment is not required.  However,
+filenames shorter than 16 bytes are NUL-padded to 16 bytes before
+being encrypted.  In addition, to reduce leakage of filename lengths
+via their ciphertexts, all filenames are NUL-padded to the next 4, 8,
+16, or 32-byte boundary (configurable).  32 is recommended since this
+provides the best confidentiality, at the cost of making directory
+entries consume slightly more space.  Note that since NUL (``\0``) is
+not otherwise a valid character in filenames, the padding will never
+produce duplicate plaintexts.
+
+Symbolic link targets are considered a type of filename and are
+encrypted in the same way as filenames in directory entries, except
+that IV reuse is not a problem as each symlink has its own inode.
+
+
diff --git a/Documentation/lustre/client_side_encryption/threat_model.txt b/Documentation/lustre/client_side_encryption/threat_model.txt
new file mode 100644
index 0000000..f9153a2
--- /dev/null
+++ b/Documentation/lustre/client_side_encryption/threat_model.txt
@@ -0,0 +1,159 @@
+===========================================
+Lustre client-level encryption threat model
+===========================================
+
+Lustre client-level encryption relies on kernel's fscrypt, and more
+precisely on v2 encryption policies.
+fscrypt is a library which filesystems can hook into to support
+transparent encryption of files and directories.
+
+As a consequence, the following threat model, extracted from
+fscrypt's and adapted, is applicable to Lustre client-level
+encryption.
+
+Ref:
+https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/fscrypt.rst
+
+Offline attacks
+---------------
+
+Provided that userspace chooses a strong encryption key, fscrypt
+protects the confidentiality of file contents and filenames in the
+event of a single point-in-time permanent offline compromise of the
+block device content.  fscrypt does not protect the confidentiality of
+non-filename metadata, e.g. file sizes, file permissions, file
+timestamps, and extended attributes.  Also, the existence and location
+of holes (unallocated blocks which logically contain all zeroes) in
+files is not protected.
+
+fscrypt is not guaranteed to protect confidentiality or authenticity
+if an attacker is able to manipulate the filesystem offline prior to
+an authorized user later accessing the filesystem.
+
+For the Lustre case, block devices are Lustre targets attached to
+the Lustre servers. Manipulating the filesystem offline means
+accessing the filesystem on these targets while Lustre is offline.
+
+Online attacks
+--------------
+
+fscrypt (and storage encryption in general) can only provide limited
+protection, if any at all, against online attacks.  In detail:
+
+Side-channel attacks
+~~~~~~~~~~~~~~~~~~~~
+
+fscrypt is only resistant to side-channel attacks, such as timing or
+electromagnetic attacks, to the extent that the underlying Linux
+Cryptographic API algorithms are.  If a vulnerable algorithm is used,
+such as a table-based implementation of AES, it may be possible for an
+attacker to mount a side channel attack against the online system.
+Side channel attacks may also be mounted against applications
+consuming decrypted data.
+
+Unauthorized file access on Lustre client
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+After an encryption key has been added, fscrypt does not hide the
+plaintext file contents or filenames from other users on the same
+system.  Instead, existing access control mechanisms such as file mode
+bits, POSIX ACLs, LSMs, or namespaces should be used for this purpose.
+
+(For the reasoning behind this, understand that while the key is
+added, the confidentiality of the data, from the perspective of the
+system itself, is *not* protected by the mathematical properties of
+encryption but rather only by the correctness of the kernel.
+Therefore, any encryption-specific access control checks would merely
+be enforced by kernel *code* and therefore would be largely redundant
+with the wide variety of access control mechanisms already available.)
+
+For the Lustre case, it means plaintext file contents or filenames are
+not hidden from other users on the same Lustre client.
+
+Lustre client kernel memory compromise
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+An attacker who compromises the system enough to read from arbitrary
+memory, e.g. by mounting a physical attack or by exploiting a kernel
+security vulnerability, can compromise all encryption keys that are
+currently in use.
+
+However, fscrypt with v2 encryption policies allows encryption keys to
+be removed from the kernel, which may protect them from later
+compromise. Key removal can be carried out by non-root users.
+
+In more detail, the FS_IOC_REMOVE_ENCRYPTION_KEY ioctl will wipe a
+master encryption key from kernel memory.  Moreover, it will try to
+evict all cached inodes which had been "unlocked" using the key,
+thereby wiping their per-file keys and making them once again appear
+"locked", i.e. in ciphertext or encrypted form.
+
+However, FS_IOC_REMOVE_ENCRYPTION_KEY has some limitations:
+
+- Per-file keys for in-use files will *not* be removed or wiped.
+  Therefore, for maximum effect, userspace should close the relevant
+  encrypted files and directories before removing a master key, as
+  well as kill any processes whose working directory is in an affected
+  encrypted directory.
+
+- The kernel cannot magically wipe copies of the master key(s) that
+  userspace might have as well.  Therefore, userspace must wipe all
+  copies of the master key(s) it makes as well.  Naturally, the same
+  also applies to all higher levels in the key hierarchy.  Userspace
+  should also follow other security precautions such as mlock()ing
+  memory containing keys to prevent it from being swapped out.
+
+- In general, decrypted contents and filenames in the kernel VFS
+  caches are freed but not wiped.  Therefore, portions thereof may be
+  recoverable from freed memory, even after the corresponding key(s)
+  were wiped.  To partially solve this, you can set
+  CONFIG_PAGE_POISONING=y in your kernel config and add page_poison=1
+  to your kernel command line.  However, this has a performance cost.
+
+- Secret keys might still exist in CPU registers, in crypto
+  accelerator hardware (if used by the crypto API to implement any of
+  the algorithms), or in other places not explicitly considered here.
+
+Lustre server kernel memory compromise
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+An attacker on a Lustre server who compromises the system enough to
+read from arbitrary memory, e.g. by mounting a physical attack or by
+exploiting a kernel security vulnerability, cannot compromise Lustre
+files content. Indeed, encryption keys are not forwarded to the Lustre
+servers, and servers do not carry out decryption or encryption.
+Moreover, RPCs received by servers contain encrypted data, which is
+put as is in blocks to be stored on disk.
+
+Per-file key compromise
+~~~~~~~~~~~~~~~~~~~~~~~
+
+With one exception, fscrypt never uses the master key(s) for
+encryption directly.  Instead, they are only used as input to a KDF
+(Key Derivation Function) to derive the actual keys.
+
+For v2 encryption policies, the KDF is HKDF-SHA512. The master key is
+passed as the "input keying material", no salt is used, and a distinct
+"application-specific information string" is used for each distinct
+key to be derived.  For example, when a per-file encryption key is
+derived, the application-specific information string is the file's
+nonce prefixed with "fscrypt\0" and a context byte.  Different context
+bytes are used for other types of derived keys.
+
+So the per-file keys used to encrypt each file individually are
+obtained from a HKDF-SHA512 mechanism that is flexible, nonreversible,
+and evenly distributes entropy from the master key.
+HKDF is also standardized and widely used by other software.
+
+As a consequence, a compromise of a per-file key only impacts the
+associated file, not the master key.
+
+Master key verification
+~~~~~~~~~~~~~~~~~~~~~~~
+
+For master keys used for v2 encryption policies, a unique 16-byte "key
+identifier" is derived using the KDF.  This value is stored in
+the clear, and is used to reliably identify the key itself.
+
+Consequently, no malicious user can associate the wrong key with
+encrypted files.
-- 
1.8.3.1

[lustre-devel] [PATCH 12/20] lustre: sec: enable client side encryption

[James Simmons]

From: Sebastien Buisson <sbuisson@ddn.com>

Enable client side encryption. By default it is activated,
letting user specifies actual encryption policy to use on
a per-directory basis. It is possible to deactivate client
side encryption by using the 'noencrypt' mount option.

Also add the test dummy encryption mode option to ease
testing.

WC-bug-id: https://jira.whamcloud.com/browse/LU-12275
Lustre-commit: 28be31137cd22 ("LU-12275 sec: enable client side encryption")
Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Reviewed-on: https://review.whamcloud.com/36143
Reviewed-by: John L. Hammond <jhammond@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 fs/lustre/include/lustre_crypto.h      | 106 ++++++++++++++++++++++++++
 fs/lustre/include/obd_support.h        |   6 ++
 fs/lustre/llite/Makefile               |   1 +
 fs/lustre/llite/crypto.c               | 132 +++++++++++++++++++++++++++++++++
 fs/lustre/llite/llite_internal.h       |  26 +++++++
 fs/lustre/llite/llite_lib.c            |  33 +++++++++
 fs/lustre/llite/super25.c              |   1 +
 include/uapi/linux/lustre/lustre_idl.h |  12 +++
 8 files changed, 317 insertions(+)
 create mode 100644 fs/lustre/include/lustre_crypto.h
 create mode 100644 fs/lustre/llite/crypto.c

diff --git a/fs/lustre/include/lustre_crypto.h b/fs/lustre/include/lustre_crypto.h
new file mode 100644
index 0000000..e9091a2
--- /dev/null
+++ b/fs/lustre/include/lustre_crypto.h
@@ -0,0 +1,106 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2019, 2020, Whamcloud.
+ */
+/*
+ * This file is part of Lustre, http://www.lustre.org/
+ */
+
+#ifndef _LUSTRE_CRYPTO_H_
+#define _LUSTRE_CRYPTO_H_
+
+struct ll_sb_info;
+bool ll_sbi_has_test_dummy_encryption(struct ll_sb_info *sbi);
+bool ll_sbi_has_encrypt(struct ll_sb_info *sbi);
+void ll_sbi_set_encrypt(struct ll_sb_info *sbi, bool set);
+
+#ifdef CONFIG_FS_ENCRYPTION
+#define __FS_HAS_ENCRYPTION 1
+#include <linux/fscrypt.h>
+
+#define llcrypt_operations	fscrypt_operations
+#define llcrypt_symlink_data	fscrypt_symlink_data
+#define llcrypt_dummy_context_enabled(inode) \
+	fscrypt_dummy_context_enabled(inode)
+#define llcrypt_has_encryption_key(inode) fscrypt_has_encryption_key(inode)
+#define llcrypt_encrypt_pagecache_blocks(page, len, offs, gfp_flags)	\
+	fscrypt_encrypt_pagecache_blocks(page, len, offs, gfp_flags)
+#define llcrypt_decrypt_pagecache_blocks(page, len, offs)	\
+	fscrypt_decrypt_pagecache_blocks(page, len, offs)
+#define llcrypt_inherit_context(parent, child, fs_data, preload)	\
+	fscrypt_inherit_context(parent, child, fs_data, preload)
+#define llcrypt_get_encryption_info(inode) fscrypt_get_encryption_info(inode)
+#define llcrypt_put_encryption_info(inode) fscrypt_put_encryption_info(inode)
+#define llcrypt_free_inode(inode)	   fscrypt_free_inode(inode)
+#define llcrypt_finalize_bounce_page(pagep)  fscrypt_finalize_bounce_page(pagep)
+#define llcrypt_file_open(inode, filp)	fscrypt_file_open(inode, filp)
+#define llcrypt_ioctl_set_policy(filp, arg)  fscrypt_ioctl_set_policy(filp, arg)
+#define llcrypt_ioctl_get_policy_ex(filp, arg)	\
+	fscrypt_ioctl_get_policy_ex(filp, arg)
+#define llcrypt_ioctl_add_key(filp, arg)	fscrypt_ioctl_add_key(filp, arg)
+#define llcrypt_ioctl_remove_key(filp, arg)  fscrypt_ioctl_remove_key(filp, arg)
+#define llcrypt_ioctl_remove_key_all_users(filp, arg)	\
+	fscrypt_ioctl_remove_key_all_users(filp, arg)
+#define llcrypt_ioctl_get_key_status(filp, arg)	\
+	fscrypt_ioctl_get_key_status(filp, arg)
+#define llcrypt_drop_inode(inode)	fscrypt_drop_inode(inode)
+#define llcrypt_prepare_rename(olddir, olddentry, newdir, newdentry, flags) \
+	fscrypt_prepare_rename(olddir, olddentry, newdir, newdentry, flags)
+#define llcrypt_prepare_link(old_dentry, dir, dentry)	\
+	fscrypt_prepare_link(old_dentry, dir, dentry)
+#define llcrypt_prepare_setattr(dentry, attr)		\
+	fscrypt_prepare_setattr(dentry, attr)
+#define llcrypt_set_ops(sb, cop)	fscrypt_set_ops(sb, cop)
+#else /* !CONFIG_FS_ENCRYPTION */
+#undef IS_ENCRYPTED
+#define IS_ENCRYPTED(x)	0
+#define llcrypt_dummy_context_enabled(inode)	NULL
+/* copied from include/linux/fscrypt.h */
+#define llcrypt_has_encryption_key(inode) false
+#define llcrypt_encrypt_pagecache_blocks(page, len, offs, gfp_flags)	\
+	ERR_PTR(-EOPNOTSUPP)
+#define llcrypt_decrypt_pagecache_blocks(page, len, offs)	-EOPNOTSUPP
+#define llcrypt_inherit_context(parent, child, fs_data, preload)     -EOPNOTSUPP
+#define llcrypt_get_encryption_info(inode)			-EOPNOTSUPP
+#define llcrypt_put_encryption_info(inode)			do {} while (0)
+#define llcrypt_free_inode(inode)				do {} while (0)
+#define llcrypt_finalize_bounce_page(pagep)			do {} while (0)
+static inline int llcrypt_file_open(struct inode *inode, struct file *filp)
+{
+	return IS_ENCRYPTED(inode) ? -EOPNOTSUPP : 0;
+}
+#define llcrypt_ioctl_set_policy(filp, arg)			-EOPNOTSUPP
+#define llcrypt_ioctl_get_policy_ex(filp, arg)			-EOPNOTSUPP
+#define llcrypt_ioctl_add_key(filp, arg)			-EOPNOTSUPP
+#define llcrypt_ioctl_remove_key(filp, arg)			-EOPNOTSUPP
+#define llcrypt_ioctl_remove_key_all_users(filp, arg)		-EOPNOTSUPP
+#define llcrypt_ioctl_get_key_status(filp, arg)			-EOPNOTSUPP
+#define llcrypt_drop_inode(inode)				 0
+#define llcrypt_prepare_rename(olddir, olddentry, newdir, newdentry, flags)    0
+#define llcrypt_prepare_link(old_dentry, dir, dentry)		 0
+#define llcrypt_prepare_setattr(dentry, attr)			 0
+#define llcrypt_set_ops(sb, cop)				do {} while (0)
+#endif /* CONFIG_FS_ENCRYPTION */
+
+#endif /* _LUSTRE_CRYPTO_H_ */
diff --git a/fs/lustre/include/obd_support.h b/fs/lustre/include/obd_support.h
index d1d21e0..b5736f8 100644
--- a/fs/lustre/include/obd_support.h
+++ b/fs/lustre/include/obd_support.h
@@ -559,6 +559,9 @@ static inline int ll_ext_to_inode_flags(int flags)
 		((flags & LUSTRE_NOATIME_FL)   ? S_NOATIME   : 0) |
 		((flags & LUSTRE_APPEND_FL)    ? S_APPEND    : 0) |
 		((flags & LUSTRE_DIRSYNC_FL)   ? S_DIRSYNC   : 0) |
+#if defined(S_ENCRYPTED)
+		((flags & LUSTRE_ENCRYPT_FL)   ? S_ENCRYPTED : 0) |
+#endif
 		((flags & LUSTRE_IMMUTABLE_FL) ? S_IMMUTABLE : 0));
 }
 
@@ -568,6 +571,9 @@ static inline int ll_inode_to_ext_flags(int iflags)
 		((iflags & S_NOATIME)   ? LUSTRE_NOATIME_FL   : 0) |
 		((iflags & S_APPEND)    ? LUSTRE_APPEND_FL    : 0) |
 		((iflags & S_DIRSYNC)   ? LUSTRE_DIRSYNC_FL   : 0) |
+#if defined(S_ENCRYPTED)
+		((iflags & S_ENCRYPTED) ? LUSTRE_ENCRYPT_FL   : 0) |
+#endif
 		((iflags & S_IMMUTABLE) ? LUSTRE_IMMUTABLE_FL : 0));
 }
 
diff --git a/fs/lustre/llite/Makefile b/fs/lustre/llite/Makefile
index c88a1b0..aa388bb6 100644
--- a/fs/lustre/llite/Makefile
+++ b/fs/lustre/llite/Makefile
@@ -10,3 +10,4 @@ lustre-y := dcache.o dir.o file.o llite_lib.o llite_nfs.o \
 	    lproc_llite.o pcc.o
 
 lustre-$(CONFIG_LUSTRE_FS_POSIX_ACL) += acl.o
+lustre-$(CONFIG_FS_ENCRYPTION)	     += crypto.o
diff --git a/fs/lustre/llite/crypto.c b/fs/lustre/llite/crypto.c
new file mode 100644
index 0000000..94189c9
--- /dev/null
+++ b/fs/lustre/llite/crypto.c
@@ -0,0 +1,132 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * GPL HEADER START
+ *
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 only,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License version 2 for more details (a copy is included
+ * in the LICENSE file that accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; If not, see
+ * http://www.gnu.org/licenses/gpl-2.0.html
+ *
+ * GPL HEADER END
+ */
+/*
+ * Copyright (c) 2019, 2020, Whamcloud.
+ */
+/*
+ * This file is part of Lustre, http://www.lustre.org/
+ */
+
+#include "llite_internal.h"
+
+static int ll_get_context(struct inode *inode, void *ctx, size_t len)
+{
+	struct dentry *dentry;
+
+	if (hlist_empty(&inode->i_dentry))
+		return -ENODATA;
+
+	hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias)
+		break;
+
+	return __vfs_getxattr(dentry, inode, LL_XATTR_NAME_ENCRYPTION_CONTEXT,
+			      ctx, len);
+}
+
+static int ll_set_context(struct inode *inode, const void *ctx, size_t len,
+			  void *fs_data)
+{
+	unsigned int ext_flags;
+	struct dentry *dentry;
+	struct md_op_data *op_data;
+	struct ptlrpc_request *req = NULL;
+	int rc;
+
+	if (inode == NULL)
+		return 0;
+
+	ext_flags = ll_inode_to_ext_flags(inode->i_flags) | LUSTRE_ENCRYPT_FL;
+	dentry = (struct dentry *)fs_data;
+
+	/* Encrypting the root directory is not allowed */
+	if (inode->i_ino == inode->i_sb->s_root->d_inode->i_ino)
+		return -EPERM;
+
+	op_data = ll_prep_md_op_data(NULL, inode, NULL, NULL, 0, 0,
+				     LUSTRE_OPC_ANY, NULL);
+	if (IS_ERR(op_data))
+		return PTR_ERR(op_data);
+
+	op_data->op_attr_flags = LUSTRE_ENCRYPT_FL;
+	op_data->op_xvalid |= OP_XVALID_FLAGS;
+	rc = md_setattr(ll_i2sbi(inode)->ll_md_exp, op_data, NULL, 0, &req);
+	ll_finish_md_op_data(op_data);
+	ptlrpc_req_finished(req);
+	if (rc)
+		return rc;
+
+	rc = __vfs_setxattr(dentry, inode, LL_XATTR_NAME_ENCRYPTION_CONTEXT,
+			    ctx, len, XATTR_CREATE);
+	if (rc)
+		return rc;
+
+	ll_update_inode_flags(inode, ext_flags);
+	return 0;
+}
+
+inline bool ll_sbi_has_test_dummy_encryption(struct ll_sb_info *sbi)
+{
+	return unlikely(sbi->ll_flags & LL_SBI_TEST_DUMMY_ENCRYPTION);
+}
+
+static bool ll_dummy_context(struct inode *inode)
+{
+	struct ll_sb_info *sbi = ll_i2sbi(inode);
+
+	return sbi ? ll_sbi_has_test_dummy_encryption(sbi) : false;
+}
+
+inline bool ll_sbi_has_encrypt(struct ll_sb_info *sbi)
+{
+	return sbi->ll_flags & LL_SBI_ENCRYPT;
+}
+
+inline void ll_sbi_set_encrypt(struct ll_sb_info *sbi, bool set)
+{
+	if (set)
+		sbi->ll_flags |= LL_SBI_ENCRYPT;
+	else
+		sbi->ll_flags &=
+			~(LL_SBI_ENCRYPT | LL_SBI_TEST_DUMMY_ENCRYPTION);
+}
+
+static bool ll_empty_dir(struct inode *inode)
+{
+	/* used by llcrypt_ioctl_set_policy(), because a policy can only be set
+	 * on an empty dir.
+	 */
+	/* Here we choose to return true, meaning we always call .set_context.
+	 * Then we rely on server side, with mdd_fix_attr() that calls
+	 * mdd_dir_is_empty() when setting encryption flag on directory.
+	 */
+	return true;
+}
+
+const struct llcrypt_operations lustre_cryptops = {
+	.key_prefix		= "lustre:",
+	.get_context		= ll_get_context,
+	.set_context		= ll_set_context,
+	.dummy_context		= ll_dummy_context,
+	.empty_dir		= ll_empty_dir,
+	.max_namelen		= NAME_MAX,
+};
diff --git a/fs/lustre/llite/llite_internal.h b/fs/lustre/llite/llite_internal.h
index be3a0b0..96585a7 100644
--- a/fs/lustre/llite/llite_internal.h
+++ b/fs/lustre/llite/llite_internal.h
@@ -44,9 +44,11 @@
 #include <lustre_mdc.h>
 #include <lustre_intent.h>
 #include <linux/compat.h>
+#include <lustre_crypto.h>
 #include <linux/namei.h>
 #include <linux/xattr.h>
 #include <linux/posix_acl_xattr.h>
+
 #include "vvp_internal.h"
 #include "range_lock.h"
 #include "pcc.h"
@@ -551,6 +553,8 @@ enum stats_track_type {
 					  */
 #define LL_SBI_TINY_WRITE	0x2000000 /* tiny write support */
 #define LL_SBI_FILE_HEAT    0x4000000 /* file heat support */
+#define LL_SBI_TEST_DUMMY_ENCRYPTION	0x8000000 /* test dummy encryption */
+#define LL_SBI_ENCRYPT	   0x10000000 /* client side encryption */
 #define LL_SBI_FLAGS {	\
 	"nolck",	\
 	"checksum",	\
@@ -579,6 +583,8 @@ enum stats_track_type {
 	"pio",		\
 	"tiny_write",	\
 	"file_heat",	\
+	"test_dummy_encryption", \
+	"noencrypt",	\
 }
 
 /*
@@ -1578,4 +1584,24 @@ static inline struct pcc_super *ll_info2pccs(struct ll_inode_info *lli)
 	return ll_i2pccs(ll_info2i(lli));
 }
 
+#ifdef CONFIG_FS_ENCRYPTION
+/* crypto.c */
+extern const struct llcrypt_operations lustre_cryptops;
+
+#else /* !CONFIG_FS_ENCRYPTION */
+inline bool ll_sbi_has_test_dummy_encryption(struct ll_sb_info *sbi)
+{
+	return false;
+}
+
+inline bool ll_sbi_has_encrypt(struct ll_sb_info *sbi)
+{
+	return false;
+}
+
+inline void ll_sbi_set_encrypt(struct ll_sb_info *sbi, bool set)
+{
+}
+#endif /* !CONFIG_FS_ENCRYPTION */
+
 #endif /* LLITE_INTERNAL_H */
diff --git a/fs/lustre/llite/llite_lib.c b/fs/lustre/llite/llite_lib.c
index 05d949b..5639b16 100644
--- a/fs/lustre/llite/llite_lib.c
+++ b/fs/lustre/llite/llite_lib.c
@@ -155,6 +155,7 @@ static struct ll_sb_info *ll_init_sbi(void)
 	sbi->ll_flags |= LL_SBI_AGL_ENABLED;
 	sbi->ll_flags |= LL_SBI_FAST_READ;
 	sbi->ll_flags |= LL_SBI_TINY_WRITE;
+	ll_sbi_set_encrypt(sbi, true);
 
 	/* root squash */
 	sbi->ll_squash.rsi_uid = 0;
@@ -551,6 +552,7 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt)
 #if THREAD_SIZE >= 8192 /*b=17630*/
 	sb->s_export_op = &lustre_export_operations;
 #endif
+	llcrypt_set_ops(sb, &lustre_cryptops);
 
 	/* make root inode
 	 * XXX: move this to after cbd setup?
@@ -923,6 +925,25 @@ static int ll_options(char *options, struct ll_sb_info *sbi)
 			*flags |= tmp;
 			goto next;
 		}
+		tmp = ll_set_opt("test_dummy_encryption", s1,
+				 LL_SBI_TEST_DUMMY_ENCRYPTION);
+		if (tmp) {
+#ifdef CONFIG_FS_ENCRYPTION
+			*flags |= tmp;
+#else
+			LCONSOLE_WARN("Test dummy encryption mount option ignored: encryption not supported\n");
+#endif
+			goto next;
+		}
+		tmp = ll_set_opt("noencrypt", s1, LL_SBI_ENCRYPT);
+		if (tmp) {
+#ifdef CONFIG_FS_ENCRYPTION
+			*flags &= ~tmp;
+#else
+			LCONSOLE_WARN("noencrypt mount option ignored: encryption not supported\n");
+#endif
+			goto next;
+		}
 		LCONSOLE_ERROR_MSG(0x152, "Unknown option '%s', won't mount.\n",
 				   s1);
 		return -EINVAL;
@@ -1558,6 +1579,8 @@ void ll_clear_inode(struct inode *inode)
 	 * cl_object still uses inode lsm.
 	 */
 	cl_inode_fini(inode);
+
+	llcrypt_put_encryption_info(inode);
 }
 
 static int ll_md_setattr(struct dentry *dentry, struct md_op_data *op_data)
@@ -1978,6 +2001,8 @@ void ll_update_inode_flags(struct inode *inode, int ext_flags)
 {
 	struct ll_inode_info *lli = ll_i2info(inode);
 
+	/* do not clear encryption flag */
+	ext_flags |= ll_inode_to_ext_flags(inode->i_flags) & LUSTRE_ENCRYPT_FL;
 	inode->i_flags = ll_ext_to_inode_flags(ext_flags);
 	if (ext_flags & LUSTRE_PROJINHERIT_FL)
 		set_bit(LLIF_PROJECT_INHERIT, &lli->lli_flags);
@@ -2702,6 +2727,14 @@ int ll_show_options(struct seq_file *seq, struct dentry *dentry)
 	if (sbi->ll_flags & LL_SBI_ALWAYS_PING)
 		seq_puts(seq, ",always_ping");
 
+	if (ll_sbi_has_test_dummy_encryption(sbi))
+		seq_puts(seq, ",test_dummy_encryption");
+
+	if (ll_sbi_has_encrypt(sbi))
+		seq_puts(seq, ",encrypt");
+	else
+		seq_puts(seq, ",noencrypt");
+
 	return 0;
 }
 
diff --git a/fs/lustre/llite/super25.c b/fs/lustre/llite/super25.c
index 006be6b..8eb3fc3 100644
--- a/fs/lustre/llite/super25.c
+++ b/fs/lustre/llite/super25.c
@@ -63,6 +63,7 @@ static void ll_inode_destroy_callback(struct rcu_head *head)
 	struct inode *inode = container_of(head, struct inode, i_rcu);
 	struct ll_inode_info *ptr = ll_i2info(inode);
 
+	llcrypt_free_inode(inode);
 	kmem_cache_free(ll_inode_cachep, ptr);
 }
 
diff --git a/include/uapi/linux/lustre/lustre_idl.h b/include/uapi/linux/lustre/lustre_idl.h
index d0b43c8..d4df989 100644
--- a/include/uapi/linux/lustre/lustre_idl.h
+++ b/include/uapi/linux/lustre/lustre_idl.h
@@ -1108,6 +1108,8 @@ struct lov_mds_md_v1 {		/* LOV EA mds/wire data (little-endian) */
 #define XATTR_NAME_HSM		"trusted.hsm"
 #define XATTR_NAME_LFSCK_NAMESPACE "trusted.lfsck_namespace"
 
+#define LL_XATTR_NAME_ENCRYPTION_CONTEXT XATTR_SECURITY_PREFIX"c"
+
 struct lov_mds_md_v3 {		/* LOV EA mds/wire data (little-endian) */
 	__u32 lmm_magic;	/* magic number = LOV_MAGIC_V3 */
 	__u32 lmm_pattern;	/* LOV_PATTERN_RAID0, LOV_PATTERN_RAID1 */
@@ -1571,6 +1573,16 @@ enum {
 	LUSTRE_TOPDIR_FL	= 0x00020000, /* Top of directory hierarchies*/
 	LUSTRE_INLINE_DATA_FL	= 0x10000000, /* Inode has inline data. */
 	LUSTRE_PROJINHERIT_FL	= 0x20000000, /* Create with parents projid */
+
+	/* These flags will not be identical to any EXT4_*_FL counterparts,
+	 * and only reserved for lustre purpose. Note: these flags might
+	 * be conflict with some of EXT4 flags, so
+	 * 1. these conflict flags needs to be removed when the flag is
+	 * wired by la_flags see osd_attr_get().
+	 * 2. If these flags needs to be stored into inode, they will be
+	 * stored in LMA. see LMAI_XXXX
+	 */
+	LUSTRE_ENCRYPT_FL       = 0x00800000, /* encrypted file */
 };
 
 /* 64 possible states */
-- 
1.8.3.1

[lustre-devel] [PATCH 13/20] lustre: ptlrpc: separate number MD and refrences for bulk

[James Simmons]

From: Alexey Lyashkov <c17817@cray.com>

Introduce a bulk desc refs, it's different from MD's count ptlrpc
expects to have events from all MD's even it's filled or not. So,
number an MD's to post is related to the requested transfer size,
not a number MD's with data.

Cray-bug-id: LUS-8139
WC-bug-id: https://jira.whamcloud.com/browse/LU-10157
Lustre-commit: 8a7f2d4b11801 ("LU-10157 ptlrpc: separate number MD and refrences for bulk")
Signed-off-by: Alexey Lyashkov <c17817@cray.com>
Reviewed-on: https://review.whamcloud.com/37386
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Shaun Tancheff <shaun.tancheff@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 fs/lustre/include/lustre_net.h |  4 +++-
 fs/lustre/ptlrpc/client.c      |  2 +-
 fs/lustre/ptlrpc/events.c      |  6 +++---
 fs/lustre/ptlrpc/niobuf.c      | 12 ++++++------
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/fs/lustre/include/lustre_net.h b/fs/lustre/include/lustre_net.h
index a94d826..18de7d9 100644
--- a/fs/lustre/include/lustre_net.h
+++ b/fs/lustre/include/lustre_net.h
@@ -1237,6 +1237,8 @@ struct ptlrpc_bulk_frag_ops {
  *  Another user is readpage for MDT.
  */
 struct ptlrpc_bulk_desc {
+	/* number MD's assigned including zero-sends */
+	unsigned int			bd_refs;
 	/** completed with failure */
 	unsigned long			bd_failure:1;
 	/** client side */
@@ -1796,7 +1798,7 @@ static inline int ptlrpc_client_bulk_active(struct ptlrpc_request *req)
 		return 1;
 
 	spin_lock(&desc->bd_lock);
-	rc = desc->bd_md_count;
+	rc = desc->bd_refs;
 	spin_unlock(&desc->bd_lock);
 	return rc;
 }
diff --git a/fs/lustre/ptlrpc/client.c b/fs/lustre/ptlrpc/client.c
index 22022ff..e69c988 100644
--- a/fs/lustre/ptlrpc/client.c
+++ b/fs/lustre/ptlrpc/client.c
@@ -244,7 +244,7 @@ void __ptlrpc_prep_bulk_page(struct ptlrpc_bulk_desc *desc,
 void ptlrpc_free_bulk(struct ptlrpc_bulk_desc *desc)
 {
 	LASSERT(desc->bd_iov_count != LI_POISON);	/* not freed already */
-	LASSERT(desc->bd_md_count == 0);		/* network hands off */
+	LASSERT(desc->bd_refs == 0);			/* network hands off */
 	LASSERT((desc->bd_export != NULL) ^ (desc->bd_import != NULL));
 	LASSERT(desc->bd_frag_ops);
 
diff --git a/fs/lustre/ptlrpc/events.c b/fs/lustre/ptlrpc/events.c
index 5e3787c..eef40b3 100644
--- a/fs/lustre/ptlrpc/events.c
+++ b/fs/lustre/ptlrpc/events.c
@@ -204,8 +204,8 @@ void client_bulk_callback(struct lnet_event *ev)
 
 	spin_lock(&desc->bd_lock);
 	req = desc->bd_req;
-	LASSERT(desc->bd_md_count > 0);
-	desc->bd_md_count--;
+	LASSERT(desc->bd_refs > 0);
+	desc->bd_refs--;
 
 	if (ev->type != LNET_EVENT_UNLINK && ev->status == 0) {
 		desc->bd_nob_transferred += ev->mlength;
@@ -223,7 +223,7 @@ void client_bulk_callback(struct lnet_event *ev)
 	/* NB don't unlock till after wakeup; desc can disappear under us
 	 * otherwise
 	 */
-	if (desc->bd_md_count == 0)
+	if (desc->bd_refs == 0)
 		ptlrpc_client_wake_req(desc->bd_req);
 
 	spin_unlock(&desc->bd_lock);
diff --git a/fs/lustre/ptlrpc/niobuf.c b/fs/lustre/ptlrpc/niobuf.c
index 331a0c8..3f8b2c6 100644
--- a/fs/lustre/ptlrpc/niobuf.c
+++ b/fs/lustre/ptlrpc/niobuf.c
@@ -169,7 +169,7 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req)
 
 	desc->bd_registered = 1;
 	desc->bd_last_mbits = mbits;
-	desc->bd_md_count = total_md;
+	desc->bd_refs = total_md;
 	md.user_ptr = &desc->bd_cbid;
 	md.handler = ptlrpc_handler;
 	md.threshold = 1;		/* PUT or GET */
@@ -211,9 +211,9 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req)
 	if (rc != 0) {
 		LASSERT(rc == -ENOMEM);
 		spin_lock(&desc->bd_lock);
-		desc->bd_md_count -= total_md - posted_md;
+		desc->bd_refs -= total_md - posted_md;
 		spin_unlock(&desc->bd_lock);
-		LASSERT(desc->bd_md_count >= 0);
+		LASSERT(desc->bd_refs >= 0);
 		mdunlink_iterate_helper(desc->bd_mds, desc->bd_md_max_brw);
 		req->rq_status = -ENOMEM;
 		desc->bd_registered = 0;
@@ -222,15 +222,15 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req)
 
 	spin_lock(&desc->bd_lock);
 	/* Holler if peer manages to touch buffers before he knows the mbits */
-	if (desc->bd_md_count != total_md)
+	if (desc->bd_refs != total_md)
 		CWARN("%s: Peer %s touched %d buffers while I registered\n",
 		      desc->bd_import->imp_obd->obd_name, libcfs_id2str(peer),
-		      total_md - desc->bd_md_count);
+		      total_md - desc->bd_refs);
 	spin_unlock(&desc->bd_lock);
 
 	CDEBUG(D_NET,
 	       "Setup %u bulk %s buffers: %u pages %u bytes, mbits x%#llx-%#llx, portal %u\n",
-	       desc->bd_md_count,
+	       desc->bd_refs,
 	       ptlrpc_is_bulk_op_get(desc->bd_type) ? "get-source" : "put-sink",
 	       desc->bd_iov_count, desc->bd_nob,
 	       desc->bd_last_mbits, req->rq_mbits, desc->bd_portal);
-- 
1.8.3.1

[lustre-devel] [PATCH 14/20] lustre: ptlrpc: fill md correctly.

[James Simmons]

From: Alexey Lyashkov <c17817@cray.com>

MD fill should limit to the overall transfer size in additional
to the number a fragment.
Let's do this.

Cray-bug-id: LUS-8139
WC-bug-id: https://jira.whamcloud.com/browse/LU-10157
Lustre-commit: e1ac9e74844dc ("LU-10157 ptlrpc: fill md correctly.")
Signed-off-by: Alexey Lyashkov <c17817@cray.com>
Reviewed-on: https://review.whamcloud.com/37387
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Shaun Tancheff <shaun.tancheff@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 fs/lustre/include/lustre_net.h |  5 +++++
 fs/lustre/ptlrpc/client.c      | 22 ++++++++++++++++++----
 fs/lustre/ptlrpc/niobuf.c      | 10 +++++-----
 fs/lustre/ptlrpc/pers.c        | 20 +++++++++++++++-----
 4 files changed, 43 insertions(+), 14 deletions(-)

diff --git a/fs/lustre/include/lustre_net.h b/fs/lustre/include/lustre_net.h
index 18de7d9..2d58f13e 100644
--- a/fs/lustre/include/lustre_net.h
+++ b/fs/lustre/include/lustre_net.h
@@ -1260,6 +1260,7 @@ struct ptlrpc_bulk_desc {
 	int				bd_max_iov;	/* allocated size of bd_iov */
 	int				bd_nob;		/* # bytes covered */
 	int				bd_nob_transferred; /* # bytes GOT/PUT */
+	unsigned int			bd_nob_last;	/* # bytes in last MD */
 
 	u64				bd_last_mbits;
 
@@ -1267,6 +1268,10 @@ struct ptlrpc_bulk_desc {
 	lnet_nid_t			bd_sender;	/* stash event::sender */
 	int				bd_md_count;	/* # valid entries in bd_mds */
 	int				bd_md_max_brw;	/* max entries in bd_mds */
+
+	/** array of offsets for each MD */
+	unsigned int			bd_mds_off[PTLRPC_BULK_OPS_COUNT];
+
 	/** array of associated MDs */
 	struct lnet_handle_md		bd_mds[PTLRPC_BULK_OPS_COUNT];
 
diff --git a/fs/lustre/ptlrpc/client.c b/fs/lustre/ptlrpc/client.c
index e69c988..c9d9fe9 100644
--- a/fs/lustre/ptlrpc/client.c
+++ b/fs/lustre/ptlrpc/client.c
@@ -147,6 +147,12 @@ struct ptlrpc_bulk_desc *ptlrpc_new_bulk(unsigned int nfrags,
 
 	LASSERT(ops->add_kiov_frag);
 
+	if (max_brw > PTLRPC_BULK_OPS_COUNT)
+		return NULL;
+
+	if (nfrags > LNET_MAX_IOV * max_brw)
+		return NULL;
+
 	desc = kzalloc(sizeof(*desc), GFP_NOFS);
 	if (!desc)
 		return NULL;
@@ -162,6 +168,7 @@ struct ptlrpc_bulk_desc *ptlrpc_new_bulk(unsigned int nfrags,
 	desc->bd_portal = portal;
 	desc->bd_type = type;
 	desc->bd_md_count = 0;
+	desc->bd_nob_last = LNET_MTU;
 	desc->bd_frag_ops = ops;
 	LASSERT(max_brw > 0);
 	desc->bd_md_max_brw = min(max_brw, PTLRPC_BULK_OPS_COUNT);
@@ -228,6 +235,15 @@ void __ptlrpc_prep_bulk_page(struct ptlrpc_bulk_desc *desc,
 
 	kiov = &desc->bd_vec[desc->bd_iov_count];
 
+	if (((desc->bd_iov_count % LNET_MAX_IOV) == 0) ||
+	    ((desc->bd_nob_last + len) > LNET_MTU)) {
+		desc->bd_mds_off[desc->bd_md_count] = desc->bd_iov_count;
+		desc->bd_md_count++;
+		desc->bd_nob_last = 0;
+		LASSERT(desc->bd_md_count <= PTLRPC_BULK_OPS_COUNT);
+	}
+
+	desc->bd_nob_last += len;
 	desc->bd_nob += len;
 
 	if (pin)
@@ -3177,9 +3193,7 @@ void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req)
 		    || req->rq_mbits == 0) {
 			req->rq_mbits = req->rq_xid;
 		} else {
-			int total_md = (bd->bd_iov_count + LNET_MAX_IOV - 1) /
-					LNET_MAX_IOV;
-			req->rq_mbits -= total_md - 1;
+			req->rq_mbits -= bd->bd_md_count - 1;
 		}
 	} else {
 		/*
@@ -3194,7 +3208,7 @@ void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req)
 	 * that server can infer the number of bulks that were prepared,
 	 * see LU-1431
 	 */
-	req->rq_mbits += DIV_ROUND_UP(bd->bd_iov_count, LNET_MAX_IOV) - 1;
+	req->rq_mbits += bd->bd_md_count - 1;
 
 	/* Set rq_xid as rq_mbits to indicate the final bulk for the old
 	 * server which does not support OBD_CONNECT_BULK_MBITS. LU-6808
diff --git a/fs/lustre/ptlrpc/niobuf.c b/fs/lustre/ptlrpc/niobuf.c
index 3f8b2c6..6fb79a2 100644
--- a/fs/lustre/ptlrpc/niobuf.c
+++ b/fs/lustre/ptlrpc/niobuf.c
@@ -131,7 +131,6 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req)
 
 	/* NB no locking required until desc is on the network */
 	LASSERT(desc->bd_nob > 0);
-	LASSERT(desc->bd_md_count == 0);
 	LASSERT(desc->bd_md_max_brw <= PTLRPC_BULK_OPS_COUNT);
 	LASSERT(desc->bd_iov_count <= PTLRPC_MAX_BRW_PAGES);
 	LASSERT(desc->bd_req);
@@ -154,9 +153,9 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req)
 	LASSERT(desc->bd_cbid.cbid_fn == client_bulk_callback);
 	LASSERT(desc->bd_cbid.cbid_arg == desc);
 
-	total_md = DIV_ROUND_UP(desc->bd_iov_count, LNET_MAX_IOV);
+	total_md = desc->bd_md_count;
 	/* rq_mbits is matchbits of the final bulk */
-	mbits = req->rq_mbits - total_md + 1;
+	mbits = req->rq_mbits - desc->bd_md_count + 1;
 
 	LASSERTF(mbits == (req->rq_mbits & PTLRPC_BULK_OPS_MASK),
 		 "first mbits = x%llu, last mbits = x%llu\n",
@@ -174,13 +173,14 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req)
 	md.handler = ptlrpc_handler;
 	md.threshold = 1;		/* PUT or GET */
 
-	for (posted_md = 0; posted_md < total_md; posted_md++, mbits++) {
+	for (posted_md = 0; posted_md < desc->bd_md_count;
+	     posted_md++, mbits++) {
 		md.options = PTLRPC_MD_OPTIONS |
 			     (ptlrpc_is_bulk_op_get(desc->bd_type) ?
 			      LNET_MD_OP_GET : LNET_MD_OP_PUT);
 		ptlrpc_fill_bulk_md(&md, desc, posted_md);
 
-		if (posted_md > 0 && posted_md + 1 == total_md &&
+		if (posted_md > 0 && posted_md + 1 == desc->bd_md_count &&
 		    OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_BULK_ATTACH)) {
 			rc = -ENOMEM;
 		} else {
diff --git a/fs/lustre/ptlrpc/pers.c b/fs/lustre/ptlrpc/pers.c
index d02b155..ecbc9d3 100644
--- a/fs/lustre/ptlrpc/pers.c
+++ b/fs/lustre/ptlrpc/pers.c
@@ -44,7 +44,7 @@
 void ptlrpc_fill_bulk_md(struct lnet_md *md, struct ptlrpc_bulk_desc *desc,
 			 int mdidx)
 {
-	int offset = mdidx * LNET_MAX_IOV;
+	unsigned int start = desc->bd_mds_off[mdidx];
 
 	BUILD_BUG_ON(PTLRPC_MAX_BRW_PAGES >= LI_POISON);
 
@@ -52,12 +52,22 @@ void ptlrpc_fill_bulk_md(struct lnet_md *md, struct ptlrpc_bulk_desc *desc,
 	LASSERT(desc->bd_iov_count <= PTLRPC_MAX_BRW_PAGES);
 	LASSERT(!(md->options & (LNET_MD_KIOV | LNET_MD_PHYS)));
 
-	md->length = max(0, desc->bd_iov_count - mdidx * LNET_MAX_IOV);
-	md->length = min_t(unsigned int, LNET_MAX_IOV, md->length);
+	/* just send a lnet header */
+	if (mdidx >= desc->bd_md_count) {
+		md->options |= LNET_MD_KIOV;
+		md->length = 0;
+		md->start = NULL;
+		return;
+	}
+
+	if (mdidx == (desc->bd_md_count - 1))
+		md->length = desc->bd_iov_count - start;
+	else
+		md->length = desc->bd_mds_off[mdidx + 1] - start;
 
 	md->options |= LNET_MD_KIOV;
 	if (desc->bd_enc_vec)
-		md->start = &desc->bd_enc_vec[offset];
+		md->start = &desc->bd_enc_vec[start];
 	else
-		md->start = &desc->bd_vec[offset];
+		md->start = &desc->bd_vec[start];
 }
-- 
1.8.3.1

[lustre-devel] [PATCH 15/20] lustre: llite: don't check mirror info for page discard

[James Simmons]

From: Bobi Jam <bobijam@whamcloud.com>

The CIT_MISC is used for locks/pages manipulation, it will not
go with full io procedure, i.e. cl_io_loop() will not be called
for it. So don't check it for plain file since the mirror info
is not initialized/set in this case.

WC-bug-id: https://jira.whamcloud.com/browse/LU-13473
Lustre-commit: d0dd744ed6ae0 ("LU-13473 llite: don't check mirror info for page discard")
Signed-off-by: Bobi Jam <bobijam@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/38307
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Wang Shilong <wshilong@ddn.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 fs/lustre/lov/lov_io.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/fs/lustre/lov/lov_io.c b/fs/lustre/lov/lov_io.c
index 615db73..99b68d8 100644
--- a/fs/lustre/lov/lov_io.c
+++ b/fs/lustre/lov/lov_io.c
@@ -299,8 +299,13 @@ static int lov_io_mirror_init(struct lov_io *lio, struct lov_object *obj,
 	int result;
 
 	if (!lov_is_flr(obj)) {
-		LASSERT(comp->lo_preferred_mirror == 0);
-		lio->lis_mirror_index = comp->lo_preferred_mirror;
+		/* only locks/pages are manipulated for CIT_MISC op, no
+		 * cl_io_loop() will be called, don't check/set mirror info.
+		 */
+		if (io->ci_type != CIT_MISC) {
+			LASSERT(comp->lo_preferred_mirror == 0);
+			lio->lis_mirror_index = comp->lo_preferred_mirror;
+		}
 		io->ci_ndelay = 0;
 		return 0;
 	}
-- 
1.8.3.1

[lustre-devel] [PATCH 16/20] lustre: sec: control client side encryption

[James Simmons]

From: Sebastien Buisson <sbuisson@ddn.com>

Client enables encryption by default. However, this should be
possible only if server side is encryption aware.
Moreover, we want to give the ability to decide which clients can
make use of encryption, by extending the nodemap mechanism with a
new 'forbid_encryption' property, set to 0 by default.

WC-bug-id: https://jira.whamcloud.com/browse/LU-12275
Lustre-commit: 3042bcd709ebf ("LU-12275 sec: control client side encryption")
Signed-off-by: Sebastien Buisson <sbuisson@ddn.com>
Reviewed-on: https://review.whamcloud.com/36433
Reviewed-by: John L. Hammond <jhammond@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 fs/lustre/llite/llite_internal.h | 17 +++++++++++++++++
 fs/lustre/llite/llite_lib.c      | 23 +++++++++++++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/fs/lustre/llite/llite_internal.h b/fs/lustre/llite/llite_internal.h
index 96585a7..a688bd8 100644
--- a/fs/lustre/llite/llite_internal.h
+++ b/fs/lustre/llite/llite_internal.h
@@ -398,6 +398,23 @@ int ll_inode_init_security(struct dentry *dentry, struct inode *inode,
 int ll_listsecurity(struct inode *inode, char *secctx_name,
 		    size_t secctx_name_size);
 
+static inline bool obd_connect_has_enc(struct obd_connect_data *data)
+{
+#ifdef CONFIG_FS_ENCRYPTION
+	return data->ocd_connect_flags & OBD_CONNECT_FLAGS2 &&
+	       data->ocd_connect_flags2 & OBD_CONNECT2_ENCRYPT;
+#else
+	return false;
+#endif
+}
+
+static inline void obd_connect_set_enc(struct obd_connect_data *data)
+{
+#ifdef HAVE_LUSTRE_CRYPTO
+	data->ocd_connect_flags2 |= OBD_CONNECT2_ENCRYPT;
+#endif
+}
+
 /*
  * Locking to guarantee consistency of non-atomic updates to long long i_size,
  * consistency between file size and KMS.
diff --git a/fs/lustre/llite/llite_lib.c b/fs/lustre/llite/llite_lib.c
index 5639b16..6cd9f7e 100644
--- a/fs/lustre/llite/llite_lib.c
+++ b/fs/lustre/llite/llite_lib.c
@@ -303,6 +303,8 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt)
 		data->ocd_connect_flags &= ~OBD_CONNECT_PINGLESS;
 
 	obd_connect_set_secctx(data);
+	if (ll_sbi_has_encrypt(sbi))
+		obd_connect_set_enc(data);
 
 #if defined(CONFIG_SECURITY)
 	data->ocd_connect_flags2 |= OBD_CONNECT2_SELINUX_POLICY;
@@ -424,6 +426,14 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt)
 	if (obd_connect_has_secctx(data))
 		sbi->ll_flags |= LL_SBI_FILE_SECCTX;
 
+	if (ll_sbi_has_encrypt(sbi) && !obd_connect_has_enc(data)) {
+		if (ll_sbi_has_test_dummy_encryption(sbi))
+			LCONSOLE_WARN("%s: server %s does not support encryption feature, encryption deactivated.\n",
+				      sbi->ll_fsname,
+				      sbi->ll_md_exp->exp_obd->obd_name);
+		ll_sbi_set_encrypt(sbi, false);
+	}
+
 	if (data->ocd_ibits_known & MDS_INODELOCK_XATTR) {
 		if (!(data->ocd_connect_flags & OBD_CONNECT_MAX_EASIZE)) {
 			LCONSOLE_INFO("%s: disabling xattr cache due to unknown maximum xattr size.\n",
@@ -486,6 +496,9 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt)
 	if (sbi->ll_flags & LL_SBI_ALWAYS_PING)
 		data->ocd_connect_flags &= ~OBD_CONNECT_PINGLESS;
 
+	if (ll_sbi_has_encrypt(sbi))
+		obd_connect_set_enc(data);
+
 	CDEBUG(D_RPCTRACE,
 	       "ocd_connect_flags: %#llx ocd_version: %d ocd_grant: %d\n",
 	       data->ocd_connect_flags,
@@ -509,6 +522,16 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt)
 		goto out_md_fid;
 	}
 
+	if (ll_sbi_has_encrypt(sbi) &&
+	    !obd_connect_has_enc(&sbi->ll_dt_obd->u.lov.lov_ocd)) {
+		if (ll_sbi_has_test_dummy_encryption(sbi))
+			LCONSOLE_WARN("%s: server %s does not support encryption feature, encryption deactivated.\n",
+				      sbi->ll_fsname, dt);
+		ll_sbi_set_encrypt(sbi, false);
+	} else if (ll_sbi_has_test_dummy_encryption(sbi)) {
+		LCONSOLE_WARN("Test dummy encryption mode enabled\n");
+	}
+
 	sbi->ll_dt_exp->exp_connect_data = *data;
 
 	/* Don't change value if it was specified in the config log */
-- 
1.8.3.1

[lustre-devel] [PATCH 17/20] lnet: fix uninitialize var in choose_ipv4_src()

[James Simmons]

From: Mr NeilBrown <neilb@suse.de>

choose_ip4_src() test "*ret" without initializing it - and callers do
not (and should not) initialize the var.

Instead of testing "*ret", test "err" - if this is non-zero (it will
be -ENOENT) we want to use the address.  If it is zero, then we only
use the address if it is on the right subnet.

Reported-by: Amir Shehata <ashehata@whamcloud.com>
Fixes: 382731148e24 ("lnet: socklnd: use interface index to track local addr")
WC-bug-id: https://jira.whamcloud.com/browse/LU-10391
Lustre-commit: 8ea895389a92d ("LU-10391 lnet: fix uninitialize var in choose_ipv4_src()")
Signed-off-by: Mr NeilBrown <neilb@suse.de>
Reviewed-on: https://review.whamcloud.com/38823
Reviewed-by: Aurelien Degremont <degremoa@amazon.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Amir Shehata <ashehata@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 net/lnet/lnet/lib-socket.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/net/lnet/lnet/lib-socket.c b/net/lnet/lnet/lib-socket.c
index a4db830..e082b52 100644
--- a/net/lnet/lnet/lib-socket.c
+++ b/net/lnet/lnet/lib-socket.c
@@ -158,7 +158,7 @@ int choose_ipv4_src(__u32 *ret, int interface, __u32 dst_ipaddr, struct net *ns)
 		goto out;
 	err = -ENOENT;
 	in_dev_for_each_ifa_rcu(ifa, in_dev) {
-		if (*ret == 0 ||
+		if (err == 0 ||
 		    ((dst_ipaddr ^ ntohl(ifa->ifa_local)) &
 		     ntohl(ifa->ifa_mask)) == 0) {
 			/* This address at least as good as what we
-- 
1.8.3.1

[lustre-devel] [PATCH 18/20] lustre: obd: Rename OS_STATE flags to OS_STATFS

[James Simmons]

From: Patrick Farrell <pfarrell@whamcloud.com>

The statfs state flags are oddly named "OS_STATE_[STATE]"
Rename them to "OS_STATFS_[STATE]" to make their role clearer
and make them easier to find.

WC-bug-id: https://jira.whamcloud.com/browse/LU-11963
Lustre-commit: 9be2db987767a ("LU-11963 obd: Rename OS_STATE flags to OS_STATFS")
Signed-off-by: Patrick Farrell <pfarrell@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/34289
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Patrick Farrell <farr0186@gmail.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 fs/lustre/include/obd_class.h           |  4 ++--
 fs/lustre/llite/llite_lib.c             |  2 +-
 fs/lustre/lmv/lmv_obd.c                 |  2 +-
 fs/lustre/ptlrpc/wiretest.c             | 28 ++++++++++++++--------------
 include/uapi/linux/lustre/lustre_user.h | 20 ++++++++++----------
 5 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/fs/lustre/include/obd_class.h b/fs/lustre/include/obd_class.h
index be73a165..746782b 100644
--- a/fs/lustre/include/obd_class.h
+++ b/fs/lustre/include/obd_class.h
@@ -942,14 +942,14 @@ static inline int obd_statfs(const struct lu_env *env, struct obd_export *exp,
 	       obd->obd_name, obd->obd_osfs_age, max_age);
 	/* ignore cache if aggregated isn't expected */
 	if (obd->obd_osfs_age < max_age ||
-	    ((obd->obd_osfs.os_state & OS_STATE_SUM) &&
+	    ((obd->obd_osfs.os_state & OS_STATFS_SUM) &&
 	     !(flags & OBD_STATFS_SUM))) {
 		/* the RPC will block anyway, so avoid sending many at once */
 		rc = mutex_lock_interruptible(&obd->obd_dev_mutex);
 		if (rc)
 			return rc;
 		if (obd->obd_osfs_age < max_age ||
-		    ((obd->obd_osfs.os_state & OS_STATE_SUM) &&
+		    ((obd->obd_osfs.os_state & OS_STATFS_SUM) &&
 		     !(flags & OBD_STATFS_SUM))) {
 			rc = OBP(obd, statfs)(env, exp, osfs, max_age, flags);
 		} else {
diff --git a/fs/lustre/llite/llite_lib.c b/fs/lustre/llite/llite_lib.c
index 6cd9f7e..aad19a2 100644
--- a/fs/lustre/llite/llite_lib.c
+++ b/fs/lustre/llite/llite_lib.c
@@ -1924,7 +1924,7 @@ int ll_statfs_internal(struct ll_sb_info *sbi, struct obd_statfs *osfs,
 	       osfs->os_bavail, osfs->os_blocks, osfs->os_ffree,
 	       osfs->os_files);
 
-	if (osfs->os_state & OS_STATE_SUM)
+	if (osfs->os_state & OS_STATFS_SUM)
 		goto out;
 
 	rc = obd_statfs(NULL, sbi->ll_dt_exp, &obd_osfs, max_age, flags);
diff --git a/fs/lustre/lmv/lmv_obd.c b/fs/lustre/lmv/lmv_obd.c
index 203a17b..c5f21cd 100644
--- a/fs/lustre/lmv/lmv_obd.c
+++ b/fs/lustre/lmv/lmv_obd.c
@@ -1255,7 +1255,7 @@ static int lmv_statfs(const struct lu_env *env, struct obd_export *exp,
 			goto out_free_temp;
 		}
 
-		if (temp->os_state & OS_STATE_SUM ||
+		if (temp->os_state & OS_STATFS_SUM ||
 		    flags == OBD_STATFS_FOR_MDT0) {
 			/* Reset to the last aggregated values
 			 * and don't sum with non-aggrated data.
diff --git a/fs/lustre/ptlrpc/wiretest.c b/fs/lustre/ptlrpc/wiretest.c
index 719d73b..74a4e940 100644
--- a/fs/lustre/ptlrpc/wiretest.c
+++ b/fs/lustre/ptlrpc/wiretest.c
@@ -1851,20 +1851,20 @@ void lustre_assert_wire_constants(void)
 		 (long long)(int)offsetof(struct obd_statfs, os_spare9));
 	LASSERTF((int)sizeof(((struct obd_statfs *)0)->os_spare9) == 4, "found %lld\n",
 		 (long long)(int)sizeof(((struct obd_statfs *)0)->os_spare9));
-	LASSERTF(OS_STATE_DEGRADED == 0x00000001UL, "found %lld\n",
-		 (long long)OS_STATE_DEGRADED);
-	LASSERTF(OS_STATE_READONLY == 0x00000002UL, "found %lld\n",
-		 (long long)OS_STATE_READONLY);
-	LASSERTF(OS_STATE_NOPRECREATE == 0x00000004UL, "found %lld\n",
-		 (long long)OS_STATE_NOPRECREATE);
-	LASSERTF(OS_STATE_ENOSPC == 0x00000020UL, "found %lld\n",
-		 (long long)OS_STATE_ENOSPC);
-	LASSERTF(OS_STATE_ENOINO == 0x00000040UL, "found %lld\n",
-		 (long long)OS_STATE_ENOINO);
-	LASSERTF(OS_STATE_SUM == 0x00000100UL, "found %lld\n",
-		 (long long)OS_STATE_SUM);
-	LASSERTF(OS_STATE_NONROT == 0x00000200UL, "found %lld\n",
-		 (long long)OS_STATE_NONROT);
+	LASSERTF(OS_STATFS_DEGRADED == 0x00000001UL, "found %lld\n",
+		 (long long)OS_STATFS_DEGRADED);
+	LASSERTF(OS_STATFS_READONLY == 0x00000002UL, "found %lld\n",
+		 (long long)OS_STATFS_READONLY);
+	LASSERTF(OS_STATFS_NOPRECREATE == 0x00000004UL, "found %lld\n",
+		 (long long)OS_STATFS_NOPRECREATE);
+	LASSERTF(OS_STATFS_ENOSPC == 0x00000020UL, "found %lld\n",
+		 (long long)OS_STATFS_ENOSPC);
+	LASSERTF(OS_STATFS_ENOINO == 0x00000040UL, "found %lld\n",
+		 (long long)OS_STATFS_ENOINO);
+	LASSERTF(OS_STATFS_SUM == 0x00000100UL, "found %lld\n",
+		 (long long)OS_STATFS_SUM);
+	LASSERTF(OS_STATFS_NONROT == 0x00000200UL, "found %lld\n",
+		 (long long)OS_STATFS_NONROT);
 
 	/* Checks for struct obd_ioobj */
 	LASSERTF((int)sizeof(struct obd_ioobj) == 24, "found %lld\n",
diff --git a/include/uapi/linux/lustre/lustre_user.h b/include/uapi/linux/lustre/lustre_user.h
index 004d373..121c064 100644
--- a/include/uapi/linux/lustre/lustre_user.h
+++ b/include/uapi/linux/lustre/lustre_user.h
@@ -105,15 +105,15 @@
 #define LUSTRE_FIEMAP_FLAGS_COMPAT (FIEMAP_FLAG_SYNC | FIEMAP_FLAG_DEVICE_ORDER)
 
 enum obd_statfs_state {
-	OS_STATE_DEGRADED	= 0x00000001, /**< RAID degraded/rebuilding */
-	OS_STATE_READONLY	= 0x00000002, /**< filesystem is read-only */
-	OS_STATE_NOPRECREATE	= 0x00000004, /**< no object precreation */
-	OS_STATE_UNUSED1	= 0x00000008, /**< obsolete 1.6, was EROFS=30 */
-	OS_STATE_UNUSED2	= 0x00000010, /**< obsolete 1.6, was EROFS=30 */
-	OS_STATE_ENOSPC		= 0x00000020, /**< not enough free space */
-	OS_STATE_ENOINO		= 0x00000040, /**< not enough inodes */
-	OS_STATE_SUM		= 0x00000100, /**< aggregated for all tagrets */
-	OS_STATE_NONROT		= 0x00000200, /**< non-rotational device */
+	OS_STATFS_DEGRADED	= 0x00000001, /**< RAID degraded/rebuilding */
+	OS_STATFS_READONLY	= 0x00000002, /**< filesystem is read-only */
+	OS_STATFS_NOPRECREATE	= 0x00000004, /**< no object precreation */
+	OS_STATFS_UNUSED1	= 0x00000008, /**< obsolete 1.6, was EROFS=30 */
+	OS_STATFS_UNUSED2	= 0x00000010, /**< obsolete 1.6, was EROFS=30 */
+	OS_STATFS_ENOSPC	= 0x00000020, /**< not enough free space */
+	OS_STATFS_ENOINO	= 0x00000040, /**< not enough inodes */
+	OS_STATFS_SUM		= 0x00000100, /**< aggregated for all tagrets */
+	OS_STATFS_NONROT	= 0x00000200, /**< non-rotational device */
 };
 
 struct obd_statfs {
@@ -127,7 +127,7 @@ struct obd_statfs {
 	__u32	os_bsize;	/* block size in bytes for os_blocks */
 	__u32	os_namelen;	/* maximum length of filename in bytes*/
 	__u64	os_maxbytes;	/* maximum object size in bytes */
-	__u32	os_state;	/**< obd_statfs_state OS_STATE_* flag */
+	__u32	os_state;	/**< obd_statfs_state OS_STATFS_* flag */
 	__u32	os_fprecreated;	/* objs available now to the caller
 				 * used in QoS code to find preferred OSTs
 				 */
-- 
1.8.3.1

[lustre-devel] [PATCH 19/20] lustre: mdc: allow setting max_mod_rpcs_in_flight larger

[James Simmons]

From: Andreas Dilger <adilger@whamcloud.com>

Allow setting mdc.*.max_mod_rpcs_in_flight > mdc.*.max_rpcs_in_flight
by increasing the latter value, rather than returning an error and
telling the user to do that.  This matches the similar behavior if
mdc.*.max_rpcs_in_flight is reduced lower than max_mod_rpcs_in_flight.

If there are multiple MDTs, the "mdc.*.max_mod_rpcs_in_flight" param
may be set from e.g. the MDT0000 config log before MDT0001 is fully
configured, catching MDT0001 with ocd_maxmodrpcs = 0 before the OCD
from the MDT has been filled in, and incorrectly trigger an error.
If seen during setup, allow ocd_maxmodrpcs = (max_rpcs_in_flight - 1),
since this will be fixed up later if mdc.*.max_rpcs_in_flight is set
smaller in the config log (if set larger it doesn't matter).

WC-bug-id: https://jira.whamcloud.com/browse/LU-13503
Lustre-commit: 6d314902e6d19 ("LU-13503 mdc: allow setting max_mod_rpcs_in_flight larger")
Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Signed-off-by: Jian Yu <yujian@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/38455
Reviewed-by: John L. Hammond <jhammond@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 fs/lustre/obdclass/genops.c | 50 +++++++++++++++++++++++++++++++++------------
 1 file changed, 37 insertions(+), 13 deletions(-)

diff --git a/fs/lustre/obdclass/genops.c b/fs/lustre/obdclass/genops.c
index 607f0d6..1647fe9 100644
--- a/fs/lustre/obdclass/genops.c
+++ b/fs/lustre/obdclass/genops.c
@@ -1324,7 +1324,6 @@ u32 obd_get_max_rpcs_in_flight(struct client_obd *cli)
 int obd_set_max_rpcs_in_flight(struct client_obd *cli, u32 max)
 {
 	struct obd_request_slot_waiter *orsw;
-	const char *type_name;
 	u32 old;
 	int diff;
 	int rc;
@@ -1333,14 +1332,18 @@ int obd_set_max_rpcs_in_flight(struct client_obd *cli, u32 max)
 	if (max > OBD_MAX_RIF_MAX || max < 1)
 		return -ERANGE;
 
-	type_name = cli->cl_import->imp_obd->obd_type->typ_name;
-	if (!strcmp(type_name, LUSTRE_MDC_NAME)) {
+	CDEBUG(D_INFO, "%s: max = %hu max_mod = %u rif = %u\n",
+	       cli->cl_import->imp_obd->obd_name, max,
+	       cli->cl_max_mod_rpcs_in_flight, cli->cl_max_rpcs_in_flight);
+
+	if (strcmp(cli->cl_import->imp_obd->obd_type->typ_name,
+		   LUSTRE_MDC_NAME) == 0) {
 		/*
 		 * adjust max_mod_rpcs_in_flight to ensure it is always
 		 * strictly lower that max_rpcs_in_flight
 		 */
 		if (max < 2) {
-			CERROR("%s: cannot set max_rpcs_in_flight to 1 because it must be higher than max_mod_rpcs_in_flight value\n",
+			CERROR("%s: cannot set mdc.*.max_rpcs_in_flight=1\n",
 			       cli->cl_import->imp_obd->obd_name);
 			return -ERANGE;
 		}
@@ -1385,22 +1388,43 @@ int obd_set_max_mod_rpcs_in_flight(struct client_obd *cli, u16 max)
 	if (max > OBD_MAX_RIF_MAX || max < 1)
 		return -ERANGE;
 
-	/* cannot exceed or equal max_rpcs_in_flight */
+	CDEBUG(D_INFO, "%s: max = %hu flags = %llx, max_mod = %u rif = %u\n",
+	       cli->cl_import->imp_obd->obd_name, max, ocd->ocd_connect_flags,
+	       ocd->ocd_maxmodrpcs, cli->cl_max_rpcs_in_flight);
+
+	if (max == OBD_MAX_RIF_MAX)
+		max = OBD_MAX_RIF_MAX - 1;
+
+	/* Cannot exceed or equal max_rpcs_in_flight.  If we are asked to
+	 * increase this value, also bump up max_rpcs_in_flight to match.
+	 */
 	if (max >= cli->cl_max_rpcs_in_flight) {
-		CERROR("%s: can't set max_mod_rpcs_in_flight to a value (%hu) higher or equal to max_rpcs_in_flight value (%u)\n",
-		       cli->cl_import->imp_obd->obd_name,
-		       max, cli->cl_max_rpcs_in_flight);
+		CDEBUG(D_INFO,
+		       "%s: increasing max_rpcs_in_flight=%hu to allow larger max_mod_rpcs_in_flight=%u\n",
+		       cli->cl_import->imp_obd->obd_name, max + 1, max);
 		return -ERANGE;
 	}
 
-	/* cannot exceed max modify RPCs in flight supported by the server */
-	ocd = &cli->cl_import->imp_connect_data;
-	if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS)
+	/* cannot exceed max modify RPCs in flight supported by the server,
+	 * but verify ocd_connect_flags is at least initialized first.  If
+	 * not, allow it and fix value later in ptlrpc_connect_set_flags().
+	 */
+	if (!ocd->ocd_connect_flags) {
+		maxmodrpcs = cli->cl_max_rpcs_in_flight - 1;
+	} else if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS) {
 		maxmodrpcs = ocd->ocd_maxmodrpcs;
-	else
+		if (maxmodrpcs == 0) { /* connection not finished yet */
+			maxmodrpcs = cli->cl_max_rpcs_in_flight - 1;
+			CDEBUG(D_INFO,
+			       "%s: partial connect, assume maxmodrpcs=%hu\n",
+			       cli->cl_import->imp_obd->obd_name, maxmodrpcs);
+		}
+	} else {
 		maxmodrpcs = 1;
+	}
+
 	if (max > maxmodrpcs) {
-		CERROR("%s: can't set max_mod_rpcs_in_flight to a value (%hu) higher than max_mod_rpcs_per_client value (%hu) returned by the server at connection\n",
+		CERROR("%s: can't set max_mod_rpcs_in_flight=%hu higher than ocd_maxmodrpcs=%hu returned by the server at connection\n",
 		       cli->cl_import->imp_obd->obd_name,
 		       max, maxmodrpcs);
 		return -ERANGE;
-- 
1.8.3.1

[lustre-devel] [PATCH 20/20] lnet: o2iblnd: 'Timed out tx' error message

[James Simmons]

From: Sonia Sharma <sharmaso@whamcloud.com>

Fix the error message in kiblnd_check_txs_locked()
to report the total RDMA time outstanding rather
than the number of seconds past the deadline.

This patch also adds time_on_activeq to struct kib_tx
so the time spent by tx in internal queue and active
queue can be tracked and reported. This would help
in diagnosing the issue.

WC-bug-id: https://jira.whamcloud.com/browse/LU-1742
Lustre-commit: 7308662efc02f ("LU-1742 o2iblnd: 'Timed out tx' error message")
Signed-off-by: Sonia Sharma <sharmaso@whamcloud.com>
Reviewed-on: https://review.whamcloud.com/33235
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Stephen Champion <stephen.champion@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Signed-off-by: James Simmons <jsimmons@infradead.org>
---
 net/lnet/klnds/o2iblnd/o2iblnd.h    |  2 ++
 net/lnet/klnds/o2iblnd/o2iblnd_cb.c | 26 ++++++++++++++++++++++----
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/net/lnet/klnds/o2iblnd/o2iblnd.h b/net/lnet/klnds/o2iblnd/o2iblnd.h
index f60a69d..dc09e5e 100644
--- a/net/lnet/klnds/o2iblnd/o2iblnd.h
+++ b/net/lnet/klnds/o2iblnd/o2iblnd.h
@@ -541,6 +541,8 @@ struct kib_tx {					/* transmit message */
 	bool			tx_gaps;
 	struct kib_fmr		tx_fmr;		/* FMR */
 	int			tx_dmadir;	/* dma direction */
+	/* time when tx added on ibc_active_txs */
+	ktime_t			tx_on_activeq;
 };
 
 struct kib_connvars {
diff --git a/net/lnet/klnds/o2iblnd/o2iblnd_cb.c b/net/lnet/klnds/o2iblnd/o2iblnd_cb.c
index 40e196d..f421cdf 100644
--- a/net/lnet/klnds/o2iblnd/o2iblnd_cb.c
+++ b/net/lnet/klnds/o2iblnd/o2iblnd_cb.c
@@ -821,6 +821,7 @@ static int kiblnd_map_tx(struct lnet_ni *ni, struct kib_tx *tx,
 	 */
 	tx->tx_sending++;
 	list_add(&tx->tx_list, &conn->ibc_active_txs);
+	tx->tx_on_activeq = ktime_get();
 
 	/* I'm still holding ibc_lock! */
 	if (conn->ibc_state != IBLND_CONN_ESTABLISHED) {
@@ -3169,6 +3170,8 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,
 static int
 kiblnd_check_txs_locked(struct kib_conn *conn, struct list_head *txs)
 {
+	bool active_txs = strcmp(kiblnd_queue2str(conn, txs),
+				 "active_txs") == 0;
 	struct kib_tx *tx;
 
 	list_for_each_entry(tx, txs, tx_list) {
@@ -3179,13 +3182,28 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid,
 			LASSERT(tx->tx_waiting || tx->tx_sending);
 		}
 
-		if (ktime_compare(ktime_get(), tx->tx_deadline) >= 0) {
-			CERROR("Timed out tx: %s, %lld seconds\n",
+		if (ktime_compare(ktime_get(), tx->tx_deadline) < 0)
+			continue;
+
+		if (!active_txs) {
+			CERROR("Timed out tx: %s, outstanding RDMA time: %lld sec\n",
 			       kiblnd_queue2str(conn, txs),
+			       *kiblnd_tunables.kib_timeout +
+			       (ktime_ms_delta(ktime_get(),
+					      tx->tx_deadline) / MSEC_PER_SEC));
+		} else {
+			CERROR("Timed out tx: %s, time in internal queue: %lld sec, time in active queue: %lld sec, outstanding RDMA time: %lld sec\n",
+			       kiblnd_queue2str(conn, txs),
+			       ktime_ms_delta(tx->tx_deadline,
+					      tx->tx_on_activeq) / MSEC_PER_SEC,
 			       ktime_ms_delta(ktime_get(),
-					      tx->tx_deadline) / MSEC_PER_SEC);
-			return 1;
+					      tx->tx_on_activeq) / MSEC_PER_SEC,
+			       *kiblnd_tunables.kib_timeout +
+			       (ktime_ms_delta(ktime_get(),
+					       tx->tx_deadline) / MSEC_PER_SEC));
 		}
+
+		return 1;
 	}
 
 	return 0;
-- 
1.8.3.1