All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH 1/2] btrfs: tree-checker: Add checker for variable length item
@ 2017-11-01 12:22 Qu Wenruo
  2017-11-01 12:22 ` [PATCH 2/2] btrfs: Cleanup existing name_len checks Qu Wenruo
  2017-11-07 20:50 ` [PATCH 1/2] btrfs: tree-checker: Add checker for variable length item David Sterba
  0 siblings, 2 replies; 8+ messages in thread
From: Qu Wenruo @ 2017-11-01 12:22 UTC (permalink / raw)
  To: linux-btrfs; +Cc: dsterba

For the following types, we have items with variable length:
(With BTRFS_ prefix and _KEY suffix snipped)

DIR_ITEM
DIR_INDEX
XATTR_ITEM
INODE_REF
INODE_EXTREF
ROOT_REF
ROOT_BACKREF

They all use @name_len to indicate their name length, and XATTR_ITEM has
extra @data_len to indicate it data length.

Despite their variable length, it's also possible to have several such
structure inside one item.

This patch will add checker to ensure:

1) No structure header and its data cross item boundary
2) Except XATTR_ITEM, no structure should have non-zero @data_len

This checker is especially useful to avoid possible access beyond
boundary for fuzzed image.

Signed-off-by: Qu Wenruo <wqu@suse.com>
---
 fs/btrfs/tree-checker.c | 123 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 123 insertions(+)

diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c
index 114fc5f0ecc5..f26e86fcbd74 100644
--- a/fs/btrfs/tree-checker.c
+++ b/fs/btrfs/tree-checker.c
@@ -222,6 +222,120 @@ static int check_csum_item(struct btrfs_root *root, struct extent_buffer *leaf,
 	return 0;
 }
 
+static u32 get_header_size(u8 type)
+{
+	switch (type) {
+	case BTRFS_DIR_ITEM_KEY:
+	case BTRFS_DIR_INDEX_KEY:
+	case BTRFS_XATTR_ITEM_KEY:
+		return sizeof(struct btrfs_dir_item);
+	case BTRFS_INODE_REF_KEY:
+		return sizeof(struct btrfs_inode_ref);
+	case BTRFS_INODE_EXTREF_KEY:
+		return sizeof(struct btrfs_inode_extref);
+	case BTRFS_ROOT_REF_KEY:
+	case BTRFS_ROOT_BACKREF_KEY:
+		return sizeof(struct btrfs_root_ref);
+	}
+	WARN_ON(1);
+	return 0;
+}
+
+static u16 get_header_namelen(struct extent_buffer *leaf, u8 type,
+			      u32 header_offset)
+{
+	/*
+	 * @header_offset is offset starts after leaf header, while the
+	 * accessors expects offset starts from leaf header.
+	 * Sowe need to adds LEAF_DATA_OFFSET here
+	 */
+	unsigned long leaf_offset = header_offset + BTRFS_LEAF_DATA_OFFSET;
+
+	switch (type) {
+	case BTRFS_DIR_ITEM_KEY:
+	case BTRFS_DIR_INDEX_KEY:
+	case BTRFS_XATTR_ITEM_KEY:
+		return btrfs_dir_name_len(leaf, (void *)leaf_offset);
+	case BTRFS_INODE_REF_KEY:
+		return btrfs_inode_ref_name_len(leaf, (void *)leaf_offset);
+	case BTRFS_INODE_EXTREF_KEY:
+		return btrfs_inode_extref_name_len(leaf, (void *)leaf_offset);
+	case BTRFS_ROOT_REF_KEY:
+	case BTRFS_ROOT_BACKREF_KEY:
+		return btrfs_root_ref_name_len(leaf, (void *)leaf_offset);
+	}
+	WARN_ON(1);
+	return 0;
+}
+
+static u16 get_header_datalen(struct extent_buffer *leaf, u8 type,
+			      unsigned long header_offset)
+{
+	/* Same as get_header_namelen */
+	unsigned long leaf_offset = header_offset + BTRFS_LEAF_DATA_OFFSET;
+
+	switch (type) {
+	case BTRFS_DIR_ITEM_KEY:
+	case BTRFS_DIR_INDEX_KEY:
+	case BTRFS_XATTR_ITEM_KEY:
+		return btrfs_dir_data_len(leaf, (void *)leaf_offset);
+	}
+	return 0;
+}
+
+/*
+ * For items with variable length, normally with namelen and tailing data.
+ * Like INODE_REF or XATTR
+ */
+static int check_variable_length_item(struct btrfs_root *root,
+				      struct extent_buffer *leaf,
+				      struct btrfs_key *key, int slot)
+{
+	u8 type = key->type;
+	u32 item_start = btrfs_item_offset_nr(leaf, slot);
+	u32 item_end = btrfs_item_end_nr(leaf, slot);
+	u32 header_size = get_header_size(type);
+	u32 total_size;
+	u32 cur = item_start;
+
+	while (cur < item_end) {
+		u32 namelen;
+		u32 datalen;
+
+		/* header itself should not cross item boundary */
+		if (cur + header_size > item_end) {
+			generic_err(root, leaf, slot,
+				"structure header crosses item boundary, have %u expect (%u, %u]",
+				cur + header_size, cur, item_end);
+			return -EUCLEAN;
+		}
+
+		namelen = get_header_namelen(leaf, type, cur);
+		datalen = get_header_datalen(leaf, type, cur);
+
+		/* Only XATTR can own data */
+		if (type != BTRFS_XATTR_ITEM_KEY && datalen) {
+			generic_err(root, leaf, slot,
+				"item has invalid data len, have %u expect 0",
+				datalen);
+			return -EUCLEAN;
+		}
+
+		total_size = header_size + namelen + datalen;
+
+		/* header and name/data should not cross item boundary */
+		if (cur + total_size > item_end) {
+			generic_err(root, leaf, slot,
+				"structure data crosses item boundary, have %u expect (%u, %u]",
+				cur + total_size, cur + header_size, item_end);
+			return -EUCLEAN;
+		}
+
+		cur += total_size;
+	}
+	return 0;
+}
+
 /*
  * Common point to switch the item-specific validation.
  */
@@ -238,6 +352,15 @@ static int check_leaf_item(struct btrfs_root *root,
 	case BTRFS_EXTENT_CSUM_KEY:
 		ret = check_csum_item(root, leaf, key, slot);
 		break;
+	case BTRFS_DIR_ITEM_KEY:
+	case BTRFS_XATTR_ITEM_KEY:
+	case BTRFS_DIR_INDEX_KEY:
+	case BTRFS_INODE_REF_KEY:
+	case BTRFS_INODE_EXTREF_KEY:
+	case BTRFS_ROOT_REF_KEY:
+	case BTRFS_ROOT_BACKREF_KEY:
+		ret = check_variable_length_item(root, leaf, key, slot);
+		break;
 	}
 	return ret;
 }
-- 
2.14.3


^ permalink raw reply related	[flat|nested] 8+ messages in thread

* [PATCH 2/2] btrfs: Cleanup existing name_len checks
  2017-11-01 12:22 [PATCH 1/2] btrfs: tree-checker: Add checker for variable length item Qu Wenruo
@ 2017-11-01 12:22 ` Qu Wenruo
  2017-11-07 20:56   ` David Sterba
  2017-11-07 20:50 ` [PATCH 1/2] btrfs: tree-checker: Add checker for variable length item David Sterba
  1 sibling, 1 reply; 8+ messages in thread
From: Qu Wenruo @ 2017-11-01 12:22 UTC (permalink / raw)
  To: linux-btrfs; +Cc: dsterba

Since tree-checker has verified leaf when reading from disk, we don't
need the existing checker.

This cleanup reverts the following commits:
fbc326159a01 ("btrfs: Verify dir_item in iterate_object_props")
64c7b01446f4 ("btrfs: Check name_len before in btrfs_del_root_ref")
488d7c456653 ("btrfs: Check name_len before reading btrfs_get_name")
59b0a7f2c7c1 ("btrfs: Check name_len before read in iterate_dir_item")
3c1d41844896 ("btrfs: Check name_len in btrfs_check_ref_name_override")
8ee8c2d62d5f ("btrfs: Verify dir_item in replay_xattr_deletes")
26a836cec2ea ("btrfs: Check name_len on add_inode_ref call path")
e79a33270d05 ("btrfs: Check name_len with boundary in verify dir_item")
19c6dcbfa746 ("btrfs: Introduce btrfs_is_name_len_valid to avoid reading beyond boundary")

Signed-off-by: Qu Wenruo <wqu@suse.com>
---
 fs/btrfs/ctree.h     |  4 +--
 fs/btrfs/dir-item.c  | 76 ++--------------------------------------------------
 fs/btrfs/export.c    |  5 ----
 fs/btrfs/inode.c     |  2 +-
 fs/btrfs/props.c     |  7 -----
 fs/btrfs/root-tree.c |  7 -----
 fs/btrfs/send.c      |  6 -----
 fs/btrfs/tree-log.c  | 43 ++++++++---------------------
 fs/btrfs/xattr.c     |  2 +-
 9 files changed, 16 insertions(+), 136 deletions(-)

diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index f7df5536ab61..b674fb244e03 100644
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
@@ -3061,14 +3061,12 @@ struct btrfs_dir_item *btrfs_lookup_xattr(struct btrfs_trans_handle *trans,
 					  const char *name, u16 name_len,
 					  int mod);
 int verify_dir_item(struct btrfs_fs_info *fs_info,
-		    struct extent_buffer *leaf, int slot,
+		    struct extent_buffer *leaf,
 		    struct btrfs_dir_item *dir_item);
 struct btrfs_dir_item *btrfs_match_dir_item_name(struct btrfs_fs_info *fs_info,
 						 struct btrfs_path *path,
 						 const char *name,
 						 int name_len);
-bool btrfs_is_name_len_valid(struct extent_buffer *leaf, int slot,
-			     unsigned long start, u16 name_len);
 
 /* orphan.c */
 int btrfs_insert_orphan_item(struct btrfs_trans_handle *trans,
diff --git a/fs/btrfs/dir-item.c b/fs/btrfs/dir-item.c
index 41cb9196eaa8..c24d615e3d7f 100644
--- a/fs/btrfs/dir-item.c
+++ b/fs/btrfs/dir-item.c
@@ -395,6 +395,8 @@ struct btrfs_dir_item *btrfs_match_dir_item_name(struct btrfs_fs_info *fs_info,
 
 	leaf = path->nodes[0];
 	dir_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dir_item);
+	if (verify_dir_item(fs_info, leaf, dir_item))
+		return NULL;
 
 	total_len = btrfs_item_size_nr(leaf, path->slots[0]);
 	while (cur < total_len) {
@@ -403,8 +405,6 @@ struct btrfs_dir_item *btrfs_match_dir_item_name(struct btrfs_fs_info *fs_info,
 			btrfs_dir_data_len(leaf, dir_item);
 		name_ptr = (unsigned long)(dir_item + 1);
 
-		if (verify_dir_item(fs_info, leaf, path->slots[0], dir_item))
-			return NULL;
 		if (btrfs_dir_name_len(leaf, dir_item) == name_len &&
 		    memcmp_extent_buffer(leaf, name, name_ptr, name_len) == 0)
 			return dir_item;
@@ -453,11 +453,9 @@ int btrfs_delete_one_dir_name(struct btrfs_trans_handle *trans,
 
 int verify_dir_item(struct btrfs_fs_info *fs_info,
 		    struct extent_buffer *leaf,
-		    int slot,
 		    struct btrfs_dir_item *dir_item)
 {
 	u16 namelen = BTRFS_NAME_LEN;
-	int ret;
 	u8 type = btrfs_dir_type(leaf, dir_item);
 
 	if (type >= BTRFS_FT_MAX) {
@@ -474,12 +472,6 @@ int verify_dir_item(struct btrfs_fs_info *fs_info,
 		return 1;
 	}
 
-	namelen = btrfs_dir_name_len(leaf, dir_item);
-	ret = btrfs_is_name_len_valid(leaf, slot,
-				      (unsigned long)(dir_item + 1), namelen);
-	if (!ret)
-		return 1;
-
 	/* BTRFS_MAX_XATTR_SIZE is the same for all dir items */
 	if ((btrfs_dir_data_len(leaf, dir_item) +
 	     btrfs_dir_name_len(leaf, dir_item)) >
@@ -492,67 +484,3 @@ int verify_dir_item(struct btrfs_fs_info *fs_info,
 
 	return 0;
 }
-
-bool btrfs_is_name_len_valid(struct extent_buffer *leaf, int slot,
-			     unsigned long start, u16 name_len)
-{
-	struct btrfs_fs_info *fs_info = leaf->fs_info;
-	struct btrfs_key key;
-	u32 read_start;
-	u32 read_end;
-	u32 item_start;
-	u32 item_end;
-	u32 size;
-	bool ret = true;
-
-	ASSERT(start > BTRFS_LEAF_DATA_OFFSET);
-
-	read_start = start - BTRFS_LEAF_DATA_OFFSET;
-	read_end = read_start + name_len;
-	item_start = btrfs_item_offset_nr(leaf, slot);
-	item_end = btrfs_item_end_nr(leaf, slot);
-
-	btrfs_item_key_to_cpu(leaf, &key, slot);
-
-	switch (key.type) {
-	case BTRFS_DIR_ITEM_KEY:
-	case BTRFS_XATTR_ITEM_KEY:
-	case BTRFS_DIR_INDEX_KEY:
-		size = sizeof(struct btrfs_dir_item);
-		break;
-	case BTRFS_INODE_REF_KEY:
-		size = sizeof(struct btrfs_inode_ref);
-		break;
-	case BTRFS_INODE_EXTREF_KEY:
-		size = sizeof(struct btrfs_inode_extref);
-		break;
-	case BTRFS_ROOT_REF_KEY:
-	case BTRFS_ROOT_BACKREF_KEY:
-		size = sizeof(struct btrfs_root_ref);
-		break;
-	default:
-		ret = false;
-		goto out;
-	}
-
-	if (read_start < item_start) {
-		ret = false;
-		goto out;
-	}
-	if (read_end > item_end) {
-		ret = false;
-		goto out;
-	}
-
-	/* there shall be item(s) before name */
-	if (read_start - item_start < size) {
-		ret = false;
-		goto out;
-	}
-
-out:
-	if (!ret)
-		btrfs_crit(fs_info, "invalid dir item name len: %u",
-			   (unsigned int)name_len);
-	return ret;
-}
diff --git a/fs/btrfs/export.c b/fs/btrfs/export.c
index fa66980726c9..87144c9f9593 100644
--- a/fs/btrfs/export.c
+++ b/fs/btrfs/export.c
@@ -282,11 +282,6 @@ static int btrfs_get_name(struct dentry *parent, char *name,
 		name_len = btrfs_inode_ref_name_len(leaf, iref);
 	}
 
-	ret = btrfs_is_name_len_valid(leaf, path->slots[0], name_ptr, name_len);
-	if (!ret) {
-		btrfs_free_path(path);
-		return -EIO;
-	}
 	read_extent_buffer(leaf, name, name_ptr, name_len);
 	btrfs_free_path(path);
 
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 7b751348abdc..976f96000b2c 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -5953,7 +5953,7 @@ static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)
 		if (btrfs_should_delete_dir_index(&del_list, found_key.offset))
 			goto next;
 		di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item);
-		if (verify_dir_item(fs_info, leaf, slot, di))
+		if (verify_dir_item(fs_info, leaf, di))
 			goto next;
 
 		name_len = btrfs_dir_name_len(leaf, di);
diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c
index f6a05f836629..c39a940d0c75 100644
--- a/fs/btrfs/props.c
+++ b/fs/btrfs/props.c
@@ -164,7 +164,6 @@ static int iterate_object_props(struct btrfs_root *root,
 						 size_t),
 				void *ctx)
 {
-	struct btrfs_fs_info *fs_info = root->fs_info;
 	int ret;
 	char *name_buf = NULL;
 	char *value_buf = NULL;
@@ -215,12 +214,6 @@ static int iterate_object_props(struct btrfs_root *root,
 			name_ptr = (unsigned long)(di + 1);
 			data_ptr = name_ptr + name_len;
 
-			if (verify_dir_item(fs_info, leaf,
-					    path->slots[0], di)) {
-				ret = -EIO;
-				goto out;
-			}
-
 			if (name_len <= XATTR_BTRFS_PREFIX_LEN ||
 			    memcmp_extent_buffer(leaf, XATTR_BTRFS_PREFIX,
 						 name_ptr,
diff --git a/fs/btrfs/root-tree.c b/fs/btrfs/root-tree.c
index 3338407ef0f0..aab0194efe46 100644
--- a/fs/btrfs/root-tree.c
+++ b/fs/btrfs/root-tree.c
@@ -387,13 +387,6 @@ int btrfs_del_root_ref(struct btrfs_trans_handle *trans,
 		WARN_ON(btrfs_root_ref_dirid(leaf, ref) != dirid);
 		WARN_ON(btrfs_root_ref_name_len(leaf, ref) != name_len);
 		ptr = (unsigned long)(ref + 1);
-		ret = btrfs_is_name_len_valid(leaf, path->slots[0], ptr,
-					      name_len);
-		if (!ret) {
-			err = -EIO;
-			goto out;
-		}
-
 		WARN_ON(memcmp_extent_buffer(leaf, name, ptr, name_len));
 		*sequence = btrfs_root_ref_sequence(leaf, ref);
 
diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index c10e4c70f02d..2a2dc603da13 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -1059,12 +1059,6 @@ static int iterate_dir_item(struct btrfs_root *root, struct btrfs_path *path,
 			}
 		}
 
-		ret = btrfs_is_name_len_valid(eb, path->slots[0],
-			  (unsigned long)(di + 1), name_len + data_len);
-		if (!ret) {
-			ret = -EIO;
-			goto out;
-		}
 		if (name_len + data_len > buf_len) {
 			buf_len = name_len + data_len;
 			if (is_vmalloc_addr(buf)) {
diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c
index aa7c71cff575..e8a4a38b190d 100644
--- a/fs/btrfs/tree-log.c
+++ b/fs/btrfs/tree-log.c
@@ -1173,19 +1173,15 @@ static inline int __add_inode_ref(struct btrfs_trans_handle *trans,
 	return 0;
 }
 
-static int extref_get_fields(struct extent_buffer *eb, int slot,
-			     unsigned long ref_ptr, u32 *namelen, char **name,
-			     u64 *index, u64 *parent_objectid)
+static int extref_get_fields(struct extent_buffer *eb, unsigned long ref_ptr,
+			     u32 *namelen, char **name, u64 *index,
+			     u64 *parent_objectid)
 {
 	struct btrfs_inode_extref *extref;
 
 	extref = (struct btrfs_inode_extref *)ref_ptr;
 
 	*namelen = btrfs_inode_extref_name_len(eb, extref);
-	if (!btrfs_is_name_len_valid(eb, slot, (unsigned long)&extref->name,
-				     *namelen))
-		return -EIO;
-
 	*name = kmalloc(*namelen, GFP_NOFS);
 	if (*name == NULL)
 		return -ENOMEM;
@@ -1200,19 +1196,14 @@ static int extref_get_fields(struct extent_buffer *eb, int slot,
 	return 0;
 }
 
-static int ref_get_fields(struct extent_buffer *eb, int slot,
-			  unsigned long ref_ptr, u32 *namelen, char **name,
-			  u64 *index)
+static int ref_get_fields(struct extent_buffer *eb, unsigned long ref_ptr,
+			  u32 *namelen, char **name, u64 *index)
 {
 	struct btrfs_inode_ref *ref;
 
 	ref = (struct btrfs_inode_ref *)ref_ptr;
 
 	*namelen = btrfs_inode_ref_name_len(eb, ref);
-	if (!btrfs_is_name_len_valid(eb, slot, (unsigned long)(ref + 1),
-				     *namelen))
-		return -EIO;
-
 	*name = kmalloc(*namelen, GFP_NOFS);
 	if (*name == NULL)
 		return -ENOMEM;
@@ -1287,8 +1278,8 @@ static noinline int add_inode_ref(struct btrfs_trans_handle *trans,
 
 	while (ref_ptr < ref_end) {
 		if (log_ref_ver) {
-			ret = extref_get_fields(eb, slot, ref_ptr, &namelen,
-					  &name, &ref_index, &parent_objectid);
+			ret = extref_get_fields(eb, ref_ptr, &namelen, &name,
+						&ref_index, &parent_objectid);
 			/*
 			 * parent object can change from one array
 			 * item to another.
@@ -1300,8 +1291,8 @@ static noinline int add_inode_ref(struct btrfs_trans_handle *trans,
 				goto out;
 			}
 		} else {
-			ret = ref_get_fields(eb, slot, ref_ptr, &namelen,
-					     &name, &ref_index);
+			ret = ref_get_fields(eb, ref_ptr, &namelen, &name,
+					     &ref_index);
 		}
 		if (ret)
 			goto out;
@@ -1848,7 +1839,7 @@ static noinline int replay_one_dir_item(struct btrfs_trans_handle *trans,
 	ptr_end = ptr + item_size;
 	while (ptr < ptr_end) {
 		di = (struct btrfs_dir_item *)ptr;
-		if (verify_dir_item(fs_info, eb, slot, di))
+		if (verify_dir_item(fs_info, eb, di))
 			return -EIO;
 		name_len = btrfs_dir_name_len(eb, di);
 		ret = replay_one_name(trans, root, path, eb, di, key);
@@ -2024,7 +2015,7 @@ static noinline int check_item_in_log(struct btrfs_trans_handle *trans,
 	ptr_end = ptr + item_size;
 	while (ptr < ptr_end) {
 		di = (struct btrfs_dir_item *)ptr;
-		if (verify_dir_item(fs_info, eb, slot, di)) {
+		if (verify_dir_item(fs_info, eb, di)) {
 			ret = -EIO;
 			goto out;
 		}
@@ -2109,7 +2100,6 @@ static int replay_xattr_deletes(struct btrfs_trans_handle *trans,
 			      struct btrfs_path *path,
 			      const u64 ino)
 {
-	struct btrfs_fs_info *fs_info = root->fs_info;
 	struct btrfs_key search_key;
 	struct btrfs_path *log_path;
 	int i;
@@ -2151,11 +2141,6 @@ static int replay_xattr_deletes(struct btrfs_trans_handle *trans,
 			u32 this_len = sizeof(*di) + name_len + data_len;
 			char *name;
 
-			ret = verify_dir_item(fs_info, path->nodes[0], i, di);
-			if (ret) {
-				ret = -EIO;
-				goto out;
-			}
 			name = kmalloc(name_len, GFP_NOFS);
 			if (!name) {
 				ret = -ENOMEM;
@@ -4572,12 +4557,6 @@ static int btrfs_check_ref_name_override(struct extent_buffer *eb,
 			this_len = sizeof(*extref) + this_name_len;
 		}
 
-		ret = btrfs_is_name_len_valid(eb, slot, name_ptr,
-					      this_name_len);
-		if (!ret) {
-			ret = -EIO;
-			goto out;
-		}
 		if (this_name_len > name_len) {
 			char *new_name;
 
diff --git a/fs/btrfs/xattr.c b/fs/btrfs/xattr.c
index 2c7e53f9ff1b..b3cbf80c5acf 100644
--- a/fs/btrfs/xattr.c
+++ b/fs/btrfs/xattr.c
@@ -336,7 +336,7 @@ ssize_t btrfs_listxattr(struct dentry *dentry, char *buffer, size_t size)
 			u32 this_len = sizeof(*di) + name_len + data_len;
 			unsigned long name_ptr = (unsigned long)(di + 1);
 
-			if (verify_dir_item(fs_info, leaf, slot, di)) {
+			if (verify_dir_item(fs_info, leaf, di)) {
 				ret = -EIO;
 				goto err;
 			}
-- 
2.14.3


^ permalink raw reply related	[flat|nested] 8+ messages in thread

* Re: [PATCH 1/2] btrfs: tree-checker: Add checker for variable length item
  2017-11-01 12:22 [PATCH 1/2] btrfs: tree-checker: Add checker for variable length item Qu Wenruo
  2017-11-01 12:22 ` [PATCH 2/2] btrfs: Cleanup existing name_len checks Qu Wenruo
@ 2017-11-07 20:50 ` David Sterba
  2017-11-08  0:13   ` Qu Wenruo
  1 sibling, 1 reply; 8+ messages in thread
From: David Sterba @ 2017-11-07 20:50 UTC (permalink / raw)
  To: Qu Wenruo; +Cc: linux-btrfs, dsterba

On Wed, Nov 01, 2017 at 08:22:12PM +0800, Qu Wenruo wrote:
> For the following types, we have items with variable length:
> (With BTRFS_ prefix and _KEY suffix snipped)
> 
> DIR_ITEM
> DIR_INDEX
> XATTR_ITEM
> INODE_REF
> INODE_EXTREF
> ROOT_REF
> ROOT_BACKREF
> 
> They all use @name_len to indicate their name length, and XATTR_ITEM has
> extra @data_len to indicate it data length.
> 
> Despite their variable length, it's also possible to have several such
> structure inside one item.
> 
> This patch will add checker to ensure:
> 
> 1) No structure header and its data cross item boundary
> 2) Except XATTR_ITEM, no structure should have non-zero @data_len
> 
> This checker is especially useful to avoid possible access beyond
> boundary for fuzzed image.
> 
> Signed-off-by: Qu Wenruo <wqu@suse.com>
> ---
>  fs/btrfs/tree-checker.c | 123 ++++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 123 insertions(+)
> 
> diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c
> index 114fc5f0ecc5..f26e86fcbd74 100644
> --- a/fs/btrfs/tree-checker.c
> +++ b/fs/btrfs/tree-checker.c
> @@ -222,6 +222,120 @@ static int check_csum_item(struct btrfs_root *root, struct extent_buffer *leaf,
>  	return 0;
>  }
>  
> +static u32 get_header_size(u8 type)
> +{
> +	switch (type) {
> +	case BTRFS_DIR_ITEM_KEY:
> +	case BTRFS_DIR_INDEX_KEY:
> +	case BTRFS_XATTR_ITEM_KEY:
> +		return sizeof(struct btrfs_dir_item);
> +	case BTRFS_INODE_REF_KEY:
> +		return sizeof(struct btrfs_inode_ref);
> +	case BTRFS_INODE_EXTREF_KEY:
> +		return sizeof(struct btrfs_inode_extref);
> +	case BTRFS_ROOT_REF_KEY:
> +	case BTRFS_ROOT_BACKREF_KEY:
> +		return sizeof(struct btrfs_root_ref);
> +	}
> +	WARN_ON(1);

The helper name suggests is generic so this warning is there to catch
new use, possibly without the matching validation function? If it is so,
I'd rather use ASSERT. If you really want a warning, then please replace
it with a message. The stacktrace from WARN_ON would not be very useful
because we know exactly how we get here, the callchaning starts in the
endio handlers.

> +	return 0;
> +}
> +
> +static u16 get_header_namelen(struct extent_buffer *leaf, u8 type,
> +			      u32 header_offset)
> +{
> +	/*
> +	 * @header_offset is offset starts after leaf header, while the
> +	 * accessors expects offset starts from leaf header.
> +	 * Sowe need to adds LEAF_DATA_OFFSET here
> +	 */

Can you please rephrase the text? (and fix the typos)

> +	unsigned long leaf_offset = header_offset + BTRFS_LEAF_DATA_OFFSET;
> +
> +	switch (type) {
> +	case BTRFS_DIR_ITEM_KEY:
> +	case BTRFS_DIR_INDEX_KEY:
> +	case BTRFS_XATTR_ITEM_KEY:
> +		return btrfs_dir_name_len(leaf, (void *)leaf_offset);
> +	case BTRFS_INODE_REF_KEY:
> +		return btrfs_inode_ref_name_len(leaf, (void *)leaf_offset);
> +	case BTRFS_INODE_EXTREF_KEY:
> +		return btrfs_inode_extref_name_len(leaf, (void *)leaf_offset);
> +	case BTRFS_ROOT_REF_KEY:
> +	case BTRFS_ROOT_BACKREF_KEY:
> +		return btrfs_root_ref_name_len(leaf, (void *)leaf_offset);
> +	}
> +	WARN_ON(1);

Similar comment as to the warning above, a message would be more
helpful to say which type is unexpected.

> +	return 0;
> +}
> +
> +static u16 get_header_datalen(struct extent_buffer *leaf, u8 type,
> +			      unsigned long header_offset)
> +{
> +	/* Same as get_header_namelen */
> +	unsigned long leaf_offset = header_offset + BTRFS_LEAF_DATA_OFFSET;
> +
> +	switch (type) {
> +	case BTRFS_DIR_ITEM_KEY:
> +	case BTRFS_DIR_INDEX_KEY:
> +	case BTRFS_XATTR_ITEM_KEY:
> +		return btrfs_dir_data_len(leaf, (void *)leaf_offset);
> +	}
> +	return 0;
> +}
> +
> +/*
> + * For items with variable length, normally with namelen and tailing data.
> + * Like INODE_REF or XATTR
> + */
> +static int check_variable_length_item(struct btrfs_root *root,
> +				      struct extent_buffer *leaf,
> +				      struct btrfs_key *key, int slot)
> +{
> +	u8 type = key->type;
> +	u32 item_start = btrfs_item_offset_nr(leaf, slot);
> +	u32 item_end = btrfs_item_end_nr(leaf, slot);
> +	u32 header_size = get_header_size(type);
> +	u32 total_size;
> +	u32 cur = item_start;
> +
> +	while (cur < item_end) {
> +		u32 namelen;
> +		u32 datalen;
> +
> +		/* header itself should not cross item boundary */
> +		if (cur + header_size > item_end) {
> +			generic_err(root, leaf, slot,
> +				"structure header crosses item boundary, have %u expect (%u, %u]",

Un-indent please. I'm slightly confused by the "(%u, %u]" notation, this
reads as an open interval from the left but I don't understand it in
this context.

> +				cur + header_size, cur, item_end);
> +			return -EUCLEAN;
> +		}
> +
> +		namelen = get_header_namelen(leaf, type, cur);
> +		datalen = get_header_datalen(leaf, type, cur);
> +
> +		/* Only XATTR can own data */
> +		if (type != BTRFS_XATTR_ITEM_KEY && datalen) {
> +			generic_err(root, leaf, slot,
> +				"item has invalid data len, have %u expect 0",
> +				datalen);
> +			return -EUCLEAN;
> +		}
> +
> +		total_size = header_size + namelen + datalen;
> +
> +		/* header and name/data should not cross item boundary */
> +		if (cur + total_size > item_end) {
> +			generic_err(root, leaf, slot,
> +				"structure data crosses item boundary, have %u expect (%u, %u]",

Same.

> +				cur + total_size, cur + header_size, item_end);
> +			return -EUCLEAN;
> +		}
> +
> +		cur += total_size;
> +	}
> +	return 0;
> +}
> +
>  /*
>   * Common point to switch the item-specific validation.
>   */
> @@ -238,6 +352,15 @@ static int check_leaf_item(struct btrfs_root *root,
>  	case BTRFS_EXTENT_CSUM_KEY:
>  		ret = check_csum_item(root, leaf, key, slot);
>  		break;
> +	case BTRFS_DIR_ITEM_KEY:
> +	case BTRFS_XATTR_ITEM_KEY:
> +	case BTRFS_DIR_INDEX_KEY:
> +	case BTRFS_INODE_REF_KEY:
> +	case BTRFS_INODE_EXTREF_KEY:
> +	case BTRFS_ROOT_REF_KEY:
> +	case BTRFS_ROOT_BACKREF_KEY:
> +		ret = check_variable_length_item(root, leaf, key, slot);
> +		break;
>  	}
>  	return ret;
>  }

Otherwise ok.

^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [PATCH 2/2] btrfs: Cleanup existing name_len checks
  2017-11-01 12:22 ` [PATCH 2/2] btrfs: Cleanup existing name_len checks Qu Wenruo
@ 2017-11-07 20:56   ` David Sterba
  2017-11-08  0:19     ` Qu Wenruo
  0 siblings, 1 reply; 8+ messages in thread
From: David Sterba @ 2017-11-07 20:56 UTC (permalink / raw)
  To: Qu Wenruo; +Cc: linux-btrfs, dsterba

On Wed, Nov 01, 2017 at 08:22:13PM +0800, Qu Wenruo wrote:
> Since tree-checker has verified leaf when reading from disk, we don't
> need the existing checker.
> 
> This cleanup reverts the following commits:
> fbc326159a01 ("btrfs: Verify dir_item in iterate_object_props")
> 64c7b01446f4 ("btrfs: Check name_len before in btrfs_del_root_ref")
> 488d7c456653 ("btrfs: Check name_len before reading btrfs_get_name")
> 59b0a7f2c7c1 ("btrfs: Check name_len before read in iterate_dir_item")
> 3c1d41844896 ("btrfs: Check name_len in btrfs_check_ref_name_override")
> 8ee8c2d62d5f ("btrfs: Verify dir_item in replay_xattr_deletes")
> 26a836cec2ea ("btrfs: Check name_len on add_inode_ref call path")
> e79a33270d05 ("btrfs: Check name_len with boundary in verify dir_item")
> 19c6dcbfa746 ("btrfs: Introduce btrfs_is_name_len_valid to avoid reading beyond boundary")

Oh well, there it goes, but I like the centralized tree checker more,
so this is a small cost.

Reviewed-by: David Sterba <dsterba@suse.com>

^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [PATCH 1/2] btrfs: tree-checker: Add checker for variable length item
  2017-11-07 20:50 ` [PATCH 1/2] btrfs: tree-checker: Add checker for variable length item David Sterba
@ 2017-11-08  0:13   ` Qu Wenruo
  2017-11-08  0:51     ` Qu Wenruo
  0 siblings, 1 reply; 8+ messages in thread
From: Qu Wenruo @ 2017-11-08  0:13 UTC (permalink / raw)
  To: dsterba, Qu Wenruo, linux-btrfs


[-- Attachment #1.1: Type: text/plain, Size: 6735 bytes --]



On 2017年11月08日 04:50, David Sterba wrote:
> On Wed, Nov 01, 2017 at 08:22:12PM +0800, Qu Wenruo wrote:
>> For the following types, we have items with variable length:
>> (With BTRFS_ prefix and _KEY suffix snipped)
>>
>> DIR_ITEM
>> DIR_INDEX
>> XATTR_ITEM
>> INODE_REF
>> INODE_EXTREF
>> ROOT_REF
>> ROOT_BACKREF
>>
>> They all use @name_len to indicate their name length, and XATTR_ITEM has
>> extra @data_len to indicate it data length.
>>
>> Despite their variable length, it's also possible to have several such
>> structure inside one item.
>>
>> This patch will add checker to ensure:
>>
>> 1) No structure header and its data cross item boundary
>> 2) Except XATTR_ITEM, no structure should have non-zero @data_len
>>
>> This checker is especially useful to avoid possible access beyond
>> boundary for fuzzed image.
>>
>> Signed-off-by: Qu Wenruo <wqu@suse.com>
>> ---
>>  fs/btrfs/tree-checker.c | 123 ++++++++++++++++++++++++++++++++++++++++++++++++
>>  1 file changed, 123 insertions(+)
>>
>> diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c
>> index 114fc5f0ecc5..f26e86fcbd74 100644
>> --- a/fs/btrfs/tree-checker.c
>> +++ b/fs/btrfs/tree-checker.c
>> @@ -222,6 +222,120 @@ static int check_csum_item(struct btrfs_root *root, struct extent_buffer *leaf,
>>  	return 0;
>>  }
>>  
>> +static u32 get_header_size(u8 type)
>> +{
>> +	switch (type) {
>> +	case BTRFS_DIR_ITEM_KEY:
>> +	case BTRFS_DIR_INDEX_KEY:
>> +	case BTRFS_XATTR_ITEM_KEY:
>> +		return sizeof(struct btrfs_dir_item);
>> +	case BTRFS_INODE_REF_KEY:
>> +		return sizeof(struct btrfs_inode_ref);
>> +	case BTRFS_INODE_EXTREF_KEY:
>> +		return sizeof(struct btrfs_inode_extref);
>> +	case BTRFS_ROOT_REF_KEY:
>> +	case BTRFS_ROOT_BACKREF_KEY:
>> +		return sizeof(struct btrfs_root_ref);
>> +	}
>> +	WARN_ON(1);
> 
> The helper name suggests is generic so this warning is there to catch
> new use, possibly without the matching validation function? If it is so,
> I'd rather use ASSERT. If you really want a warning, then please replace
> it with a message. The stacktrace from WARN_ON would not be very useful
> because we know exactly how we get here, the callchaning starts in the
> endio handlers.

Please ignore this patch, as there is v2 patch which doesn't use such
centralized check.

Thanks,
Qu

> 
>> +	return 0;
>> +}
>> +
>> +static u16 get_header_namelen(struct extent_buffer *leaf, u8 type,
>> +			      u32 header_offset)
>> +{
>> +	/*
>> +	 * @header_offset is offset starts after leaf header, while the
>> +	 * accessors expects offset starts from leaf header.
>> +	 * Sowe need to adds LEAF_DATA_OFFSET here
>> +	 */
> 
> Can you please rephrase the text? (and fix the typos)
> 
>> +	unsigned long leaf_offset = header_offset + BTRFS_LEAF_DATA_OFFSET;
>> +
>> +	switch (type) {
>> +	case BTRFS_DIR_ITEM_KEY:
>> +	case BTRFS_DIR_INDEX_KEY:
>> +	case BTRFS_XATTR_ITEM_KEY:
>> +		return btrfs_dir_name_len(leaf, (void *)leaf_offset);
>> +	case BTRFS_INODE_REF_KEY:
>> +		return btrfs_inode_ref_name_len(leaf, (void *)leaf_offset);
>> +	case BTRFS_INODE_EXTREF_KEY:
>> +		return btrfs_inode_extref_name_len(leaf, (void *)leaf_offset);
>> +	case BTRFS_ROOT_REF_KEY:
>> +	case BTRFS_ROOT_BACKREF_KEY:
>> +		return btrfs_root_ref_name_len(leaf, (void *)leaf_offset);
>> +	}
>> +	WARN_ON(1);
> 
> Similar comment as to the warning above, a message would be more
> helpful to say which type is unexpected.
> 
>> +	return 0;
>> +}
>> +
>> +static u16 get_header_datalen(struct extent_buffer *leaf, u8 type,
>> +			      unsigned long header_offset)
>> +{
>> +	/* Same as get_header_namelen */
>> +	unsigned long leaf_offset = header_offset + BTRFS_LEAF_DATA_OFFSET;
>> +
>> +	switch (type) {
>> +	case BTRFS_DIR_ITEM_KEY:
>> +	case BTRFS_DIR_INDEX_KEY:
>> +	case BTRFS_XATTR_ITEM_KEY:
>> +		return btrfs_dir_data_len(leaf, (void *)leaf_offset);
>> +	}
>> +	return 0;
>> +}
>> +
>> +/*
>> + * For items with variable length, normally with namelen and tailing data.
>> + * Like INODE_REF or XATTR
>> + */
>> +static int check_variable_length_item(struct btrfs_root *root,
>> +				      struct extent_buffer *leaf,
>> +				      struct btrfs_key *key, int slot)
>> +{
>> +	u8 type = key->type;
>> +	u32 item_start = btrfs_item_offset_nr(leaf, slot);
>> +	u32 item_end = btrfs_item_end_nr(leaf, slot);
>> +	u32 header_size = get_header_size(type);
>> +	u32 total_size;
>> +	u32 cur = item_start;
>> +
>> +	while (cur < item_end) {
>> +		u32 namelen;
>> +		u32 datalen;
>> +
>> +		/* header itself should not cross item boundary */
>> +		if (cur + header_size > item_end) {
>> +			generic_err(root, leaf, slot,
>> +				"structure header crosses item boundary, have %u expect (%u, %u]",
> 
> Un-indent please. I'm slightly confused by the "(%u, %u]" notation, this
> reads as an open interval from the left but I don't understand it in
> this context.
> 
>> +				cur + header_size, cur, item_end);
>> +			return -EUCLEAN;
>> +		}
>> +
>> +		namelen = get_header_namelen(leaf, type, cur);
>> +		datalen = get_header_datalen(leaf, type, cur);
>> +
>> +		/* Only XATTR can own data */
>> +		if (type != BTRFS_XATTR_ITEM_KEY && datalen) {
>> +			generic_err(root, leaf, slot,
>> +				"item has invalid data len, have %u expect 0",
>> +				datalen);
>> +			return -EUCLEAN;
>> +		}
>> +
>> +		total_size = header_size + namelen + datalen;
>> +
>> +		/* header and name/data should not cross item boundary */
>> +		if (cur + total_size > item_end) {
>> +			generic_err(root, leaf, slot,
>> +				"structure data crosses item boundary, have %u expect (%u, %u]",
> 
> Same.
> 
>> +				cur + total_size, cur + header_size, item_end);
>> +			return -EUCLEAN;
>> +		}
>> +
>> +		cur += total_size;
>> +	}
>> +	return 0;
>> +}
>> +
>>  /*
>>   * Common point to switch the item-specific validation.
>>   */
>> @@ -238,6 +352,15 @@ static int check_leaf_item(struct btrfs_root *root,
>>  	case BTRFS_EXTENT_CSUM_KEY:
>>  		ret = check_csum_item(root, leaf, key, slot);
>>  		break;
>> +	case BTRFS_DIR_ITEM_KEY:
>> +	case BTRFS_XATTR_ITEM_KEY:
>> +	case BTRFS_DIR_INDEX_KEY:
>> +	case BTRFS_INODE_REF_KEY:
>> +	case BTRFS_INODE_EXTREF_KEY:
>> +	case BTRFS_ROOT_REF_KEY:
>> +	case BTRFS_ROOT_BACKREF_KEY:
>> +		ret = check_variable_length_item(root, leaf, key, slot);
>> +		break;
>>  	}
>>  	return ret;
>>  }
> 
> Otherwise ok.
> --
> To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 520 bytes --]

^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [PATCH 2/2] btrfs: Cleanup existing name_len checks
  2017-11-07 20:56   ` David Sterba
@ 2017-11-08  0:19     ` Qu Wenruo
  0 siblings, 0 replies; 8+ messages in thread
From: Qu Wenruo @ 2017-11-08  0:19 UTC (permalink / raw)
  To: dsterba, Qu Wenruo, linux-btrfs


[-- Attachment #1.1: Type: text/plain, Size: 1479 bytes --]



On 2017年11月08日 04:56, David Sterba wrote:
> On Wed, Nov 01, 2017 at 08:22:13PM +0800, Qu Wenruo wrote:
>> Since tree-checker has verified leaf when reading from disk, we don't
>> need the existing checker.
>>
>> This cleanup reverts the following commits:
>> fbc326159a01 ("btrfs: Verify dir_item in iterate_object_props")
>> 64c7b01446f4 ("btrfs: Check name_len before in btrfs_del_root_ref")
>> 488d7c456653 ("btrfs: Check name_len before reading btrfs_get_name")
>> 59b0a7f2c7c1 ("btrfs: Check name_len before read in iterate_dir_item")
>> 3c1d41844896 ("btrfs: Check name_len in btrfs_check_ref_name_override")
>> 8ee8c2d62d5f ("btrfs: Verify dir_item in replay_xattr_deletes")
>> 26a836cec2ea ("btrfs: Check name_len on add_inode_ref call path")
>> e79a33270d05 ("btrfs: Check name_len with boundary in verify dir_item")
>> 19c6dcbfa746 ("btrfs: Introduce btrfs_is_name_len_valid to avoid reading beyond boundary")
> 
> Oh well, there it goes, but I like the centralized tree checker more,
> so this is a small cost.
> 
> Reviewed-by: David Sterba <dsterba@suse.com>

Sorry, in v2 patch this patch is also affected.

Since in v2, even verify_dir_item() also get removed, so it's no longer
a big revert patch.

Thanks,
Qu

> --
> To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 520 bytes --]

^ permalink raw reply	[flat|nested] 8+ messages in thread

* Re: [PATCH 1/2] btrfs: tree-checker: Add checker for variable length item
  2017-11-08  0:13   ` Qu Wenruo
@ 2017-11-08  0:51     ` Qu Wenruo
  0 siblings, 0 replies; 8+ messages in thread
From: Qu Wenruo @ 2017-11-08  0:51 UTC (permalink / raw)
  To: dsterba, Qu Wenruo, linux-btrfs


[-- Attachment #1.1: Type: text/plain, Size: 7288 bytes --]



On 2017年11月08日 08:13, Qu Wenruo wrote:
> 
> 
> On 2017年11月08日 04:50, David Sterba wrote:
>> On Wed, Nov 01, 2017 at 08:22:12PM +0800, Qu Wenruo wrote:
>>> For the following types, we have items with variable length:
>>> (With BTRFS_ prefix and _KEY suffix snipped)
>>>
>>> DIR_ITEM
>>> DIR_INDEX
>>> XATTR_ITEM
>>> INODE_REF
>>> INODE_EXTREF
>>> ROOT_REF
>>> ROOT_BACKREF
>>>
>>> They all use @name_len to indicate their name length, and XATTR_ITEM has
>>> extra @data_len to indicate it data length.
>>>
>>> Despite their variable length, it's also possible to have several such
>>> structure inside one item.
>>>
>>> This patch will add checker to ensure:
>>>
>>> 1) No structure header and its data cross item boundary
>>> 2) Except XATTR_ITEM, no structure should have non-zero @data_len
>>>
>>> This checker is especially useful to avoid possible access beyond
>>> boundary for fuzzed image.
>>>
>>> Signed-off-by: Qu Wenruo <wqu@suse.com>
>>> ---
>>>  fs/btrfs/tree-checker.c | 123 ++++++++++++++++++++++++++++++++++++++++++++++++
>>>  1 file changed, 123 insertions(+)
>>>
>>> diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c
>>> index 114fc5f0ecc5..f26e86fcbd74 100644
>>> --- a/fs/btrfs/tree-checker.c
>>> +++ b/fs/btrfs/tree-checker.c
>>> @@ -222,6 +222,120 @@ static int check_csum_item(struct btrfs_root *root, struct extent_buffer *leaf,
>>>  	return 0;
>>>  }
>>>  
>>> +static u32 get_header_size(u8 type)
>>> +{
>>> +	switch (type) {
>>> +	case BTRFS_DIR_ITEM_KEY:
>>> +	case BTRFS_DIR_INDEX_KEY:
>>> +	case BTRFS_XATTR_ITEM_KEY:
>>> +		return sizeof(struct btrfs_dir_item);
>>> +	case BTRFS_INODE_REF_KEY:
>>> +		return sizeof(struct btrfs_inode_ref);
>>> +	case BTRFS_INODE_EXTREF_KEY:
>>> +		return sizeof(struct btrfs_inode_extref);
>>> +	case BTRFS_ROOT_REF_KEY:
>>> +	case BTRFS_ROOT_BACKREF_KEY:
>>> +		return sizeof(struct btrfs_root_ref);
>>> +	}
>>> +	WARN_ON(1);
>>
>> The helper name suggests is generic so this warning is there to catch
>> new use, possibly without the matching validation function? If it is so,
>> I'd rather use ASSERT. If you really want a warning, then please replace
>> it with a message. The stacktrace from WARN_ON would not be very useful
>> because we know exactly how we get here, the callchaning starts in the
>> endio handlers.
> 
> Please ignore this patch, as there is v2 patch which doesn't use such
> centralized check.

The already submitted patchset is "[PATCH 0/3] tree-checker bug fix and
enhancement."

And patch "[PATCH 2/3] btrfs: tree-checker: Add checker for dir item"
was originally going to replace it.

I'll update the patchset to address the indent comment in v2 patchset.

Thanks,
Qu
> 
> Thanks,
> Qu
> 
>>
>>> +	return 0;
>>> +}
>>> +
>>> +static u16 get_header_namelen(struct extent_buffer *leaf, u8 type,
>>> +			      u32 header_offset)
>>> +{
>>> +	/*
>>> +	 * @header_offset is offset starts after leaf header, while the
>>> +	 * accessors expects offset starts from leaf header.
>>> +	 * Sowe need to adds LEAF_DATA_OFFSET here
>>> +	 */
>>
>> Can you please rephrase the text? (and fix the typos)
>>
>>> +	unsigned long leaf_offset = header_offset + BTRFS_LEAF_DATA_OFFSET;
>>> +
>>> +	switch (type) {
>>> +	case BTRFS_DIR_ITEM_KEY:
>>> +	case BTRFS_DIR_INDEX_KEY:
>>> +	case BTRFS_XATTR_ITEM_KEY:
>>> +		return btrfs_dir_name_len(leaf, (void *)leaf_offset);
>>> +	case BTRFS_INODE_REF_KEY:
>>> +		return btrfs_inode_ref_name_len(leaf, (void *)leaf_offset);
>>> +	case BTRFS_INODE_EXTREF_KEY:
>>> +		return btrfs_inode_extref_name_len(leaf, (void *)leaf_offset);
>>> +	case BTRFS_ROOT_REF_KEY:
>>> +	case BTRFS_ROOT_BACKREF_KEY:
>>> +		return btrfs_root_ref_name_len(leaf, (void *)leaf_offset);
>>> +	}
>>> +	WARN_ON(1);
>>
>> Similar comment as to the warning above, a message would be more
>> helpful to say which type is unexpected.
>>
>>> +	return 0;
>>> +}
>>> +
>>> +static u16 get_header_datalen(struct extent_buffer *leaf, u8 type,
>>> +			      unsigned long header_offset)
>>> +{
>>> +	/* Same as get_header_namelen */
>>> +	unsigned long leaf_offset = header_offset + BTRFS_LEAF_DATA_OFFSET;
>>> +
>>> +	switch (type) {
>>> +	case BTRFS_DIR_ITEM_KEY:
>>> +	case BTRFS_DIR_INDEX_KEY:
>>> +	case BTRFS_XATTR_ITEM_KEY:
>>> +		return btrfs_dir_data_len(leaf, (void *)leaf_offset);
>>> +	}
>>> +	return 0;
>>> +}
>>> +
>>> +/*
>>> + * For items with variable length, normally with namelen and tailing data.
>>> + * Like INODE_REF or XATTR
>>> + */
>>> +static int check_variable_length_item(struct btrfs_root *root,
>>> +				      struct extent_buffer *leaf,
>>> +				      struct btrfs_key *key, int slot)
>>> +{
>>> +	u8 type = key->type;
>>> +	u32 item_start = btrfs_item_offset_nr(leaf, slot);
>>> +	u32 item_end = btrfs_item_end_nr(leaf, slot);
>>> +	u32 header_size = get_header_size(type);
>>> +	u32 total_size;
>>> +	u32 cur = item_start;
>>> +
>>> +	while (cur < item_end) {
>>> +		u32 namelen;
>>> +		u32 datalen;
>>> +
>>> +		/* header itself should not cross item boundary */
>>> +		if (cur + header_size > item_end) {
>>> +			generic_err(root, leaf, slot,
>>> +				"structure header crosses item boundary, have %u expect (%u, %u]",
>>
>> Un-indent please. I'm slightly confused by the "(%u, %u]" notation, this
>> reads as an open interval from the left but I don't understand it in
>> this context.
>>
>>> +				cur + header_size, cur, item_end);
>>> +			return -EUCLEAN;
>>> +		}
>>> +
>>> +		namelen = get_header_namelen(leaf, type, cur);
>>> +		datalen = get_header_datalen(leaf, type, cur);
>>> +
>>> +		/* Only XATTR can own data */
>>> +		if (type != BTRFS_XATTR_ITEM_KEY && datalen) {
>>> +			generic_err(root, leaf, slot,
>>> +				"item has invalid data len, have %u expect 0",
>>> +				datalen);
>>> +			return -EUCLEAN;
>>> +		}
>>> +
>>> +		total_size = header_size + namelen + datalen;
>>> +
>>> +		/* header and name/data should not cross item boundary */
>>> +		if (cur + total_size > item_end) {
>>> +			generic_err(root, leaf, slot,
>>> +				"structure data crosses item boundary, have %u expect (%u, %u]",
>>
>> Same.
>>
>>> +				cur + total_size, cur + header_size, item_end);
>>> +			return -EUCLEAN;
>>> +		}
>>> +
>>> +		cur += total_size;
>>> +	}
>>> +	return 0;
>>> +}
>>> +
>>>  /*
>>>   * Common point to switch the item-specific validation.
>>>   */
>>> @@ -238,6 +352,15 @@ static int check_leaf_item(struct btrfs_root *root,
>>>  	case BTRFS_EXTENT_CSUM_KEY:
>>>  		ret = check_csum_item(root, leaf, key, slot);
>>>  		break;
>>> +	case BTRFS_DIR_ITEM_KEY:
>>> +	case BTRFS_XATTR_ITEM_KEY:
>>> +	case BTRFS_DIR_INDEX_KEY:
>>> +	case BTRFS_INODE_REF_KEY:
>>> +	case BTRFS_INODE_EXTREF_KEY:
>>> +	case BTRFS_ROOT_REF_KEY:
>>> +	case BTRFS_ROOT_BACKREF_KEY:
>>> +		ret = check_variable_length_item(root, leaf, key, slot);
>>> +		break;
>>>  	}
>>>  	return ret;
>>>  }
>>
>> Otherwise ok.
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>>
> 


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 520 bytes --]

^ permalink raw reply	[flat|nested] 8+ messages in thread

* [PATCH 2/2] btrfs: Cleanup existing name_len checks
  2017-11-01 12:14 Qu Wenruo
@ 2017-11-01 12:14 ` Qu Wenruo
  0 siblings, 0 replies; 8+ messages in thread
From: Qu Wenruo @ 2017-11-01 12:14 UTC (permalink / raw)
  To: linux-btrfs; +Cc: dsterba

Since tree-checker has verified leaf when reading from disk, we don't
need the existing checker.

This cleanup reverts the following commits:
fbc326159a01 ("btrfs: Verify dir_item in iterate_object_props")
64c7b01446f4 ("btrfs: Check name_len before in btrfs_del_root_ref")
488d7c456653 ("btrfs: Check name_len before reading btrfs_get_name")
59b0a7f2c7c1 ("btrfs: Check name_len before read in iterate_dir_item")
3c1d41844896 ("btrfs: Check name_len in btrfs_check_ref_name_override")
8ee8c2d62d5f ("btrfs: Verify dir_item in replay_xattr_deletes")
26a836cec2ea ("btrfs: Check name_len on add_inode_ref call path")
e79a33270d05 ("btrfs: Check name_len with boundary in verify dir_item")
19c6dcbfa746 ("btrfs: Introduce btrfs_is_name_len_valid to avoid reading beyond boundary")
---
 fs/btrfs/ctree.h     |  4 +--
 fs/btrfs/dir-item.c  | 76 ++--------------------------------------------------
 fs/btrfs/export.c    |  5 ----
 fs/btrfs/inode.c     |  2 +-
 fs/btrfs/props.c     |  7 -----
 fs/btrfs/root-tree.c |  7 -----
 fs/btrfs/send.c      |  6 -----
 fs/btrfs/tree-log.c  | 43 ++++++++---------------------
 fs/btrfs/xattr.c     |  2 +-
 9 files changed, 16 insertions(+), 136 deletions(-)

diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index f7df5536ab61..b674fb244e03 100644
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
@@ -3061,14 +3061,12 @@ struct btrfs_dir_item *btrfs_lookup_xattr(struct btrfs_trans_handle *trans,
 					  const char *name, u16 name_len,
 					  int mod);
 int verify_dir_item(struct btrfs_fs_info *fs_info,
-		    struct extent_buffer *leaf, int slot,
+		    struct extent_buffer *leaf,
 		    struct btrfs_dir_item *dir_item);
 struct btrfs_dir_item *btrfs_match_dir_item_name(struct btrfs_fs_info *fs_info,
 						 struct btrfs_path *path,
 						 const char *name,
 						 int name_len);
-bool btrfs_is_name_len_valid(struct extent_buffer *leaf, int slot,
-			     unsigned long start, u16 name_len);
 
 /* orphan.c */
 int btrfs_insert_orphan_item(struct btrfs_trans_handle *trans,
diff --git a/fs/btrfs/dir-item.c b/fs/btrfs/dir-item.c
index 41cb9196eaa8..c24d615e3d7f 100644
--- a/fs/btrfs/dir-item.c
+++ b/fs/btrfs/dir-item.c
@@ -395,6 +395,8 @@ struct btrfs_dir_item *btrfs_match_dir_item_name(struct btrfs_fs_info *fs_info,
 
 	leaf = path->nodes[0];
 	dir_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dir_item);
+	if (verify_dir_item(fs_info, leaf, dir_item))
+		return NULL;
 
 	total_len = btrfs_item_size_nr(leaf, path->slots[0]);
 	while (cur < total_len) {
@@ -403,8 +405,6 @@ struct btrfs_dir_item *btrfs_match_dir_item_name(struct btrfs_fs_info *fs_info,
 			btrfs_dir_data_len(leaf, dir_item);
 		name_ptr = (unsigned long)(dir_item + 1);
 
-		if (verify_dir_item(fs_info, leaf, path->slots[0], dir_item))
-			return NULL;
 		if (btrfs_dir_name_len(leaf, dir_item) == name_len &&
 		    memcmp_extent_buffer(leaf, name, name_ptr, name_len) == 0)
 			return dir_item;
@@ -453,11 +453,9 @@ int btrfs_delete_one_dir_name(struct btrfs_trans_handle *trans,
 
 int verify_dir_item(struct btrfs_fs_info *fs_info,
 		    struct extent_buffer *leaf,
-		    int slot,
 		    struct btrfs_dir_item *dir_item)
 {
 	u16 namelen = BTRFS_NAME_LEN;
-	int ret;
 	u8 type = btrfs_dir_type(leaf, dir_item);
 
 	if (type >= BTRFS_FT_MAX) {
@@ -474,12 +472,6 @@ int verify_dir_item(struct btrfs_fs_info *fs_info,
 		return 1;
 	}
 
-	namelen = btrfs_dir_name_len(leaf, dir_item);
-	ret = btrfs_is_name_len_valid(leaf, slot,
-				      (unsigned long)(dir_item + 1), namelen);
-	if (!ret)
-		return 1;
-
 	/* BTRFS_MAX_XATTR_SIZE is the same for all dir items */
 	if ((btrfs_dir_data_len(leaf, dir_item) +
 	     btrfs_dir_name_len(leaf, dir_item)) >
@@ -492,67 +484,3 @@ int verify_dir_item(struct btrfs_fs_info *fs_info,
 
 	return 0;
 }
-
-bool btrfs_is_name_len_valid(struct extent_buffer *leaf, int slot,
-			     unsigned long start, u16 name_len)
-{
-	struct btrfs_fs_info *fs_info = leaf->fs_info;
-	struct btrfs_key key;
-	u32 read_start;
-	u32 read_end;
-	u32 item_start;
-	u32 item_end;
-	u32 size;
-	bool ret = true;
-
-	ASSERT(start > BTRFS_LEAF_DATA_OFFSET);
-
-	read_start = start - BTRFS_LEAF_DATA_OFFSET;
-	read_end = read_start + name_len;
-	item_start = btrfs_item_offset_nr(leaf, slot);
-	item_end = btrfs_item_end_nr(leaf, slot);
-
-	btrfs_item_key_to_cpu(leaf, &key, slot);
-
-	switch (key.type) {
-	case BTRFS_DIR_ITEM_KEY:
-	case BTRFS_XATTR_ITEM_KEY:
-	case BTRFS_DIR_INDEX_KEY:
-		size = sizeof(struct btrfs_dir_item);
-		break;
-	case BTRFS_INODE_REF_KEY:
-		size = sizeof(struct btrfs_inode_ref);
-		break;
-	case BTRFS_INODE_EXTREF_KEY:
-		size = sizeof(struct btrfs_inode_extref);
-		break;
-	case BTRFS_ROOT_REF_KEY:
-	case BTRFS_ROOT_BACKREF_KEY:
-		size = sizeof(struct btrfs_root_ref);
-		break;
-	default:
-		ret = false;
-		goto out;
-	}
-
-	if (read_start < item_start) {
-		ret = false;
-		goto out;
-	}
-	if (read_end > item_end) {
-		ret = false;
-		goto out;
-	}
-
-	/* there shall be item(s) before name */
-	if (read_start - item_start < size) {
-		ret = false;
-		goto out;
-	}
-
-out:
-	if (!ret)
-		btrfs_crit(fs_info, "invalid dir item name len: %u",
-			   (unsigned int)name_len);
-	return ret;
-}
diff --git a/fs/btrfs/export.c b/fs/btrfs/export.c
index fa66980726c9..87144c9f9593 100644
--- a/fs/btrfs/export.c
+++ b/fs/btrfs/export.c
@@ -282,11 +282,6 @@ static int btrfs_get_name(struct dentry *parent, char *name,
 		name_len = btrfs_inode_ref_name_len(leaf, iref);
 	}
 
-	ret = btrfs_is_name_len_valid(leaf, path->slots[0], name_ptr, name_len);
-	if (!ret) {
-		btrfs_free_path(path);
-		return -EIO;
-	}
 	read_extent_buffer(leaf, name, name_ptr, name_len);
 	btrfs_free_path(path);
 
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 7b751348abdc..976f96000b2c 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -5953,7 +5953,7 @@ static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)
 		if (btrfs_should_delete_dir_index(&del_list, found_key.offset))
 			goto next;
 		di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item);
-		if (verify_dir_item(fs_info, leaf, slot, di))
+		if (verify_dir_item(fs_info, leaf, di))
 			goto next;
 
 		name_len = btrfs_dir_name_len(leaf, di);
diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c
index f6a05f836629..c39a940d0c75 100644
--- a/fs/btrfs/props.c
+++ b/fs/btrfs/props.c
@@ -164,7 +164,6 @@ static int iterate_object_props(struct btrfs_root *root,
 						 size_t),
 				void *ctx)
 {
-	struct btrfs_fs_info *fs_info = root->fs_info;
 	int ret;
 	char *name_buf = NULL;
 	char *value_buf = NULL;
@@ -215,12 +214,6 @@ static int iterate_object_props(struct btrfs_root *root,
 			name_ptr = (unsigned long)(di + 1);
 			data_ptr = name_ptr + name_len;
 
-			if (verify_dir_item(fs_info, leaf,
-					    path->slots[0], di)) {
-				ret = -EIO;
-				goto out;
-			}
-
 			if (name_len <= XATTR_BTRFS_PREFIX_LEN ||
 			    memcmp_extent_buffer(leaf, XATTR_BTRFS_PREFIX,
 						 name_ptr,
diff --git a/fs/btrfs/root-tree.c b/fs/btrfs/root-tree.c
index 3338407ef0f0..aab0194efe46 100644
--- a/fs/btrfs/root-tree.c
+++ b/fs/btrfs/root-tree.c
@@ -387,13 +387,6 @@ int btrfs_del_root_ref(struct btrfs_trans_handle *trans,
 		WARN_ON(btrfs_root_ref_dirid(leaf, ref) != dirid);
 		WARN_ON(btrfs_root_ref_name_len(leaf, ref) != name_len);
 		ptr = (unsigned long)(ref + 1);
-		ret = btrfs_is_name_len_valid(leaf, path->slots[0], ptr,
-					      name_len);
-		if (!ret) {
-			err = -EIO;
-			goto out;
-		}
-
 		WARN_ON(memcmp_extent_buffer(leaf, name, ptr, name_len));
 		*sequence = btrfs_root_ref_sequence(leaf, ref);
 
diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c
index c10e4c70f02d..2a2dc603da13 100644
--- a/fs/btrfs/send.c
+++ b/fs/btrfs/send.c
@@ -1059,12 +1059,6 @@ static int iterate_dir_item(struct btrfs_root *root, struct btrfs_path *path,
 			}
 		}
 
-		ret = btrfs_is_name_len_valid(eb, path->slots[0],
-			  (unsigned long)(di + 1), name_len + data_len);
-		if (!ret) {
-			ret = -EIO;
-			goto out;
-		}
 		if (name_len + data_len > buf_len) {
 			buf_len = name_len + data_len;
 			if (is_vmalloc_addr(buf)) {
diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c
index aa7c71cff575..e8a4a38b190d 100644
--- a/fs/btrfs/tree-log.c
+++ b/fs/btrfs/tree-log.c
@@ -1173,19 +1173,15 @@ static inline int __add_inode_ref(struct btrfs_trans_handle *trans,
 	return 0;
 }
 
-static int extref_get_fields(struct extent_buffer *eb, int slot,
-			     unsigned long ref_ptr, u32 *namelen, char **name,
-			     u64 *index, u64 *parent_objectid)
+static int extref_get_fields(struct extent_buffer *eb, unsigned long ref_ptr,
+			     u32 *namelen, char **name, u64 *index,
+			     u64 *parent_objectid)
 {
 	struct btrfs_inode_extref *extref;
 
 	extref = (struct btrfs_inode_extref *)ref_ptr;
 
 	*namelen = btrfs_inode_extref_name_len(eb, extref);
-	if (!btrfs_is_name_len_valid(eb, slot, (unsigned long)&extref->name,
-				     *namelen))
-		return -EIO;
-
 	*name = kmalloc(*namelen, GFP_NOFS);
 	if (*name == NULL)
 		return -ENOMEM;
@@ -1200,19 +1196,14 @@ static int extref_get_fields(struct extent_buffer *eb, int slot,
 	return 0;
 }
 
-static int ref_get_fields(struct extent_buffer *eb, int slot,
-			  unsigned long ref_ptr, u32 *namelen, char **name,
-			  u64 *index)
+static int ref_get_fields(struct extent_buffer *eb, unsigned long ref_ptr,
+			  u32 *namelen, char **name, u64 *index)
 {
 	struct btrfs_inode_ref *ref;
 
 	ref = (struct btrfs_inode_ref *)ref_ptr;
 
 	*namelen = btrfs_inode_ref_name_len(eb, ref);
-	if (!btrfs_is_name_len_valid(eb, slot, (unsigned long)(ref + 1),
-				     *namelen))
-		return -EIO;
-
 	*name = kmalloc(*namelen, GFP_NOFS);
 	if (*name == NULL)
 		return -ENOMEM;
@@ -1287,8 +1278,8 @@ static noinline int add_inode_ref(struct btrfs_trans_handle *trans,
 
 	while (ref_ptr < ref_end) {
 		if (log_ref_ver) {
-			ret = extref_get_fields(eb, slot, ref_ptr, &namelen,
-					  &name, &ref_index, &parent_objectid);
+			ret = extref_get_fields(eb, ref_ptr, &namelen, &name,
+						&ref_index, &parent_objectid);
 			/*
 			 * parent object can change from one array
 			 * item to another.
@@ -1300,8 +1291,8 @@ static noinline int add_inode_ref(struct btrfs_trans_handle *trans,
 				goto out;
 			}
 		} else {
-			ret = ref_get_fields(eb, slot, ref_ptr, &namelen,
-					     &name, &ref_index);
+			ret = ref_get_fields(eb, ref_ptr, &namelen, &name,
+					     &ref_index);
 		}
 		if (ret)
 			goto out;
@@ -1848,7 +1839,7 @@ static noinline int replay_one_dir_item(struct btrfs_trans_handle *trans,
 	ptr_end = ptr + item_size;
 	while (ptr < ptr_end) {
 		di = (struct btrfs_dir_item *)ptr;
-		if (verify_dir_item(fs_info, eb, slot, di))
+		if (verify_dir_item(fs_info, eb, di))
 			return -EIO;
 		name_len = btrfs_dir_name_len(eb, di);
 		ret = replay_one_name(trans, root, path, eb, di, key);
@@ -2024,7 +2015,7 @@ static noinline int check_item_in_log(struct btrfs_trans_handle *trans,
 	ptr_end = ptr + item_size;
 	while (ptr < ptr_end) {
 		di = (struct btrfs_dir_item *)ptr;
-		if (verify_dir_item(fs_info, eb, slot, di)) {
+		if (verify_dir_item(fs_info, eb, di)) {
 			ret = -EIO;
 			goto out;
 		}
@@ -2109,7 +2100,6 @@ static int replay_xattr_deletes(struct btrfs_trans_handle *trans,
 			      struct btrfs_path *path,
 			      const u64 ino)
 {
-	struct btrfs_fs_info *fs_info = root->fs_info;
 	struct btrfs_key search_key;
 	struct btrfs_path *log_path;
 	int i;
@@ -2151,11 +2141,6 @@ static int replay_xattr_deletes(struct btrfs_trans_handle *trans,
 			u32 this_len = sizeof(*di) + name_len + data_len;
 			char *name;
 
-			ret = verify_dir_item(fs_info, path->nodes[0], i, di);
-			if (ret) {
-				ret = -EIO;
-				goto out;
-			}
 			name = kmalloc(name_len, GFP_NOFS);
 			if (!name) {
 				ret = -ENOMEM;
@@ -4572,12 +4557,6 @@ static int btrfs_check_ref_name_override(struct extent_buffer *eb,
 			this_len = sizeof(*extref) + this_name_len;
 		}
 
-		ret = btrfs_is_name_len_valid(eb, slot, name_ptr,
-					      this_name_len);
-		if (!ret) {
-			ret = -EIO;
-			goto out;
-		}
 		if (this_name_len > name_len) {
 			char *new_name;
 
diff --git a/fs/btrfs/xattr.c b/fs/btrfs/xattr.c
index 2c7e53f9ff1b..b3cbf80c5acf 100644
--- a/fs/btrfs/xattr.c
+++ b/fs/btrfs/xattr.c
@@ -336,7 +336,7 @@ ssize_t btrfs_listxattr(struct dentry *dentry, char *buffer, size_t size)
 			u32 this_len = sizeof(*di) + name_len + data_len;
 			unsigned long name_ptr = (unsigned long)(di + 1);
 
-			if (verify_dir_item(fs_info, leaf, slot, di)) {
+			if (verify_dir_item(fs_info, leaf, di)) {
 				ret = -EIO;
 				goto err;
 			}
-- 
2.14.3


^ permalink raw reply related	[flat|nested] 8+ messages in thread

end of thread, other threads:[~2017-11-08  0:52 UTC | newest]

Thread overview: 8+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2017-11-01 12:22 [PATCH 1/2] btrfs: tree-checker: Add checker for variable length item Qu Wenruo
2017-11-01 12:22 ` [PATCH 2/2] btrfs: Cleanup existing name_len checks Qu Wenruo
2017-11-07 20:56   ` David Sterba
2017-11-08  0:19     ` Qu Wenruo
2017-11-07 20:50 ` [PATCH 1/2] btrfs: tree-checker: Add checker for variable length item David Sterba
2017-11-08  0:13   ` Qu Wenruo
2017-11-08  0:51     ` Qu Wenruo
  -- strict thread matches above, loose matches on Subject: below --
2017-11-01 12:14 Qu Wenruo
2017-11-01 12:14 ` [PATCH 2/2] btrfs: Cleanup existing name_len checks Qu Wenruo

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.