[PATCH 1/9] e2fsck: fix e2fsck_allocate_memory() overflow

[PATCH 1/9] e2fsck: fix e2fsck_allocate_memory() overflow

[Andreas Dilger]

e2fsck_allocate_memory() takes an "unsigned int size" argument, which
will overflow for allocations above 4GB.  This happens for dir_info
and dx_dir_info arrays when there are more than 350M directories in a
filesystem, and for the dblist array above 180M directories.

There is also a risk of overflow during the binary search in both
e2fsck_get_dir_info() and e2fsck_get_dx_dir_info() when the midpoint
of the array is calculated, if there would be more than 2B directories
in the filesystem and working above the half way point.

Also, in some places inode numbers are "int" instead of "ext2_ino_t",
which can also cause problems with the array size calculations, and
makes it hard to identify where inode numbers are used.

Fix e2fsck_allocate_memory() to take an "unsigned long" argument to
match ext2fs_get_mem(), so that it can do single memory allocations
over 4GB.

Fix e2fsck_get_dir_info() and e2fsck_get_dx_dir_info() to temporarily
use an unsigned long long value to calculate the midpoint (which will
always fit into an ext2_ino_t again afterward).

Change variables that hold inode numbers to be ext2_ino_t, and print
them as unsigned values instead of printing negative inode numbers.

Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Shilong Wang <wshilong@ddn.com>
Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197
---
 e2fsck/dirinfo.c    | 37 +++++++++++++++++++------------------
 e2fsck/dx_dirinfo.c | 11 ++++++-----
 e2fsck/e2fsck.h     |  8 ++++----
 e2fsck/logfile.c    |  2 +-
 e2fsck/pass2.c      | 11 ++++++-----
 e2fsck/util.c       |  7 +++----
 6 files changed, 39 insertions(+), 37 deletions(-)

diff --git a/e2fsck/dirinfo.c b/e2fsck/dirinfo.c
index cceadac..49d624c 100644
--- a/e2fsck/dirinfo.c
+++ b/e2fsck/dirinfo.c
@@ -17,8 +17,8 @@
 #include <ext2fs/tdb.h>
 
 struct dir_info_db {
-	int		count;
-	int		size;
+	ext2_ino_t	count;
+	ext2_ino_t	size;
 	struct dir_info *array;
 	struct dir_info *last_lookup;
 #ifdef CONFIG_TDB
@@ -28,7 +28,7 @@ struct dir_info_db {
 };
 
 struct dir_info_iter {
-	int	i;
+	ext2_ino_t	i;
 #ifdef CONFIG_TDB
 	TDB_DATA	tdb_iter;
 #endif
@@ -46,7 +46,7 @@ static void e2fsck_put_dir_info(e2fsck_t ctx, struct dir_info *dir);
 static void setup_tdb(e2fsck_t ctx, ext2_ino_t num_dirs)
 {
 	struct dir_info_db	*db = ctx->dir_info;
-	unsigned int		threshold;
+	ext2_ino_t		threshold;
 	errcode_t		retval;
 	mode_t			save_umask;
 	char			*tdb_dir, uuid[40];
@@ -130,12 +130,12 @@ static void setup_db(e2fsck_t ctx)
 void e2fsck_add_dir_info(e2fsck_t ctx, ext2_ino_t ino, ext2_ino_t parent)
 {
 	struct dir_info		*dir, *old_array;
-	int			i, j;
+	ext2_ino_t		i, j;
 	errcode_t		retval;
 	unsigned long		old_size;
 
 #ifdef DIRINFO_DEBUG
-	printf("add_dir_info for inode (%lu, %lu)...\n", ino, parent);
+	printf("add_dir_info for inode (%u, %u)...\n", ino, parent);
 #endif
 	if (!ctx->dir_info)
 		setup_db(ctx);
@@ -149,7 +149,7 @@ void e2fsck_add_dir_info(e2fsck_t ctx, ext2_ino_t ino, ext2_ino_t parent)
 					   &ctx->dir_info->array);
 		if (retval) {
 			fprintf(stderr, "Couldn't reallocate dir_info "
-				"structure to %d entries\n",
+				"structure to %u entries\n",
 				ctx->dir_info->size);
 			fatal_error(ctx, 0);
 			ctx->dir_info->size -= 10;
@@ -204,13 +204,13 @@ void e2fsck_add_dir_info(e2fsck_t ctx, ext2_ino_t ino, ext2_ino_t parent)
 static struct dir_info *e2fsck_get_dir_info(e2fsck_t ctx, ext2_ino_t ino)
 {
 	struct dir_info_db	*db = ctx->dir_info;
-	int			low, high, mid;
+	ext2_ino_t low, high, mid;
 
 	if (!db)
 		return 0;
 
 #ifdef DIRINFO_DEBUG
-	printf("e2fsck_get_dir_info %d...", ino);
+	printf("e2fsck_get_dir_info %u...", ino);
 #endif
 
 #ifdef CONFIG_TDB
@@ -235,7 +235,7 @@ static struct dir_info *e2fsck_get_dir_info(e2fsck_t ctx, ext2_ino_t ino)
 		ret_dir_info.dotdot = buf->dotdot;
 		ret_dir_info.parent = buf->parent;
 #ifdef DIRINFO_DEBUG
-		printf("(%d,%d,%d)\n", ino, buf->dotdot, buf->parent);
+		printf("(%u,%u,%u)\n", ino, buf->dotdot, buf->parent);
 #endif
 		free(data.dptr);
 		return &ret_dir_info;
@@ -246,10 +246,10 @@ static struct dir_info *e2fsck_get_dir_info(e2fsck_t ctx, ext2_ino_t ino)
 		return db->last_lookup;
 
 	low = 0;
-	high = ctx->dir_info->count-1;
+	high = ctx->dir_info->count - 1;
 	if (ino == ctx->dir_info->array[low].ino) {
 #ifdef DIRINFO_DEBUG
-		printf("(%d,%d,%d)\n", ino,
+		printf("(%u,%u,%u)\n", ino,
 		       ctx->dir_info->array[low].dotdot,
 		       ctx->dir_info->array[low].parent);
 #endif
@@ -257,7 +257,7 @@ static struct dir_info *e2fsck_get_dir_info(e2fsck_t ctx, ext2_ino_t ino)
 	}
 	if (ino == ctx->dir_info->array[high].ino) {
 #ifdef DIRINFO_DEBUG
-		printf("(%d,%d,%d)\n", ino,
+		printf("(%u,%u,%u)\n", ino,
 		       ctx->dir_info->array[high].dotdot,
 		       ctx->dir_info->array[high].parent);
 #endif
@@ -265,12 +265,13 @@ static struct dir_info *e2fsck_get_dir_info(e2fsck_t ctx, ext2_ino_t ino)
 	}
 
 	while (low < high) {
-		mid = (low+high)/2;
+		/* sum may overflow, but result will fit into mid again */
+		mid = (unsigned long long)(low + high) / 2;
 		if (mid == low || mid == high)
 			break;
 		if (ino == ctx->dir_info->array[mid].ino) {
 #ifdef DIRINFO_DEBUG
-			printf("(%d,%d,%d)\n", ino,
+			printf("(%u,%u,%u)\n", ino,
 			       ctx->dir_info->array[mid].dotdot,
 			       ctx->dir_info->array[mid].parent);
 #endif
@@ -294,7 +295,7 @@ static void e2fsck_put_dir_info(e2fsck_t ctx EXT2FS_NO_TDB_UNUSED,
 #endif
 
 #ifdef DIRINFO_DEBUG
-	printf("e2fsck_put_dir_info (%d, %d, %d)...", dir->ino, dir->dotdot,
+	printf("e2fsck_put_dir_info (%u, %u, %u)...", dir->ino, dir->dotdot,
 	       dir->parent);
 #endif
 
@@ -329,7 +330,7 @@ void e2fsck_free_dir_info(e2fsck_t ctx)
 			if (unlink(ctx->dir_info->tdb_fn) < 0)
 				com_err("e2fsck_free_dir_info", errno,
 					_("while freeing dir_info tdb file"));
-			free(ctx->dir_info->tdb_fn);
+			ext2fs_free_mem(&ctx->dir_info->tdb_fn);
 		}
 #endif
 		if (ctx->dir_info->array)
@@ -412,7 +413,7 @@ struct dir_info *e2fsck_dir_info_iter(e2fsck_t ctx, struct dir_info_iter *iter)
 		return 0;
 
 #ifdef DIRINFO_DEBUG
-	printf("iter(%d, %d, %d)...", ctx->dir_info->array[iter->i].ino,
+	printf("iter(%u, %u, %u)...", ctx->dir_info->array[iter->i].ino,
 	       ctx->dir_info->array[iter->i].dotdot,
 	       ctx->dir_info->array[iter->i].parent);
 #endif
diff --git a/e2fsck/dx_dirinfo.c b/e2fsck/dx_dirinfo.c
index c0b0e9a..89672b7 100644
--- a/e2fsck/dx_dirinfo.c
+++ b/e2fsck/dx_dirinfo.c
@@ -17,7 +17,7 @@ void e2fsck_add_dx_dir(e2fsck_t ctx, ext2_ino_t ino, struct ext2_inode *inode,
 		       int num_blocks)
 {
 	struct dx_dir_info *dir;
-	int		i, j;
+	ext2_ino_t	i, j;
 	errcode_t	retval;
 	unsigned long	old_size;
 
@@ -41,7 +41,7 @@ void e2fsck_add_dx_dir(e2fsck_t ctx, ext2_ino_t ino, struct ext2_inode *inode,
 					   &ctx->dx_dir_info);
 		if (retval) {
 			fprintf(stderr, "Couldn't reallocate dx_dir_info "
-				"structure to %d entries\n",
+				"structure to %u entries\n",
 				ctx->dx_dir_info_size);
 			fatal_error(ctx, 0);
 			ctx->dx_dir_info_size -= 10;
@@ -86,7 +86,7 @@ void e2fsck_add_dx_dir(e2fsck_t ctx, ext2_ino_t ino, struct ext2_inode *inode,
  */
 struct dx_dir_info *e2fsck_get_dx_dir_info(e2fsck_t ctx, ext2_ino_t ino)
 {
-	int	low, high, mid;
+	ext2_ino_t low, high, mid;
 
 	low = 0;
 	high = ctx->dx_dir_info_count-1;
@@ -98,7 +98,8 @@ struct dx_dir_info *e2fsck_get_dx_dir_info(e2fsck_t ctx, ext2_ino_t ino)
 		return &ctx->dx_dir_info[high];
 
 	while (low < high) {
-		mid = (low+high)/2;
+		/* sum may overflow, but result will fit into mid again */
+		mid = (unsigned long long)(low + high) / 2;
 		if (mid == low || mid == high)
 			break;
 		if (ino == ctx->dx_dir_info[mid].ino)
@@ -116,8 +117,8 @@ struct dx_dir_info *e2fsck_get_dx_dir_info(e2fsck_t ctx, ext2_ino_t ino)
  */
 void e2fsck_free_dx_dir_info(e2fsck_t ctx)
 {
-	int	i;
 	struct dx_dir_info *dir;
+	ext2_ino_t i;
 
 	if (ctx->dx_dir_info) {
 		dir = ctx->dx_dir_info;
diff --git a/e2fsck/e2fsck.h b/e2fsck/e2fsck.h
index 2d359b3..253f8b5 100644
--- a/e2fsck/e2fsck.h
+++ b/e2fsck/e2fsck.h
@@ -318,9 +318,9 @@ struct e2fsck_struct {
 	/*
 	 * Indexed directory information
 	 */
-	int		dx_dir_info_count;
-	int		dx_dir_info_size;
-	struct dx_dir_info *dx_dir_info;
+	ext2_ino_t		dx_dir_info_count;
+	ext2_ino_t		dx_dir_info_size;
+	struct dx_dir_info	*dx_dir_info;
 
 	/*
 	 * Directories to hash
@@ -595,7 +595,7 @@ int check_backup_super_block(e2fsck_t ctx);
 void check_resize_inode(e2fsck_t ctx);
 
 /* util.c */
-extern void *e2fsck_allocate_memory(e2fsck_t ctx, unsigned int size,
+extern void *e2fsck_allocate_memory(e2fsck_t ctx, unsigned long size,
 				    const char *description);
 extern int ask(e2fsck_t ctx, const char * string, int def);
 extern int ask_yn(e2fsck_t ctx, const char * string, int def);
diff --git a/e2fsck/logfile.c b/e2fsck/logfile.c
index 3eeefd1..63e9a12 100644
--- a/e2fsck/logfile.c
+++ b/e2fsck/logfile.c
@@ -353,7 +353,7 @@ void set_up_logging(e2fsck_t ctx)
 					    ctx->problem_log_fn);
 }
 #else
-void *e2fsck_allocate_memory(e2fsck_t ctx, unsigned int size,
+void *e2fsck_allocate_memory(e2fsck_t ctx, unsigned long size,
 			     const char *description)
 {
 	void *ret;
diff --git a/e2fsck/pass2.c b/e2fsck/pass2.c
index 8b40e93..5c3f7b8 100644
--- a/e2fsck/pass2.c
+++ b/e2fsck/pass2.c
@@ -131,7 +131,8 @@ void e2fsck_pass2(e2fsck_t ctx)
 	struct dx_dir_info	*dx_dir;
 	struct dx_dirblock_info	*dx_db;
 	int			b;
-	int			i, depth;
+	ext2_ino_t		i;
+	int			depth;
 	problem_t		code;
 	int			bad_dir;
 	int (*check_dir_func)(ext2_filsys fs,
@@ -586,10 +587,10 @@ static void parse_int_node(ext2_filsys fs,
 #ifdef DX_DEBUG
 		printf("Root node dump:\n");
 		printf("\t Reserved zero: %u\n", root->reserved_zero);
-		printf("\t Hash Version: %d\n", root->hash_version);
-		printf("\t Info length: %d\n", root->info_length);
-		printf("\t Indirect levels: %d\n", root->indirect_levels);
-		printf("\t Flags: %d\n", root->unused_flags);
+		printf("\t Hash Version: %u\n", root->hash_version);
+		printf("\t Info length: %u\n", root->info_length);
+		printf("\t Indirect levels: %u\n", root->indirect_levels);
+		printf("\t Flags: %x\n", root->unused_flags);
 #endif
 
 		ent = (struct ext2_dx_entry *) (block_buf + 24 + root->info_length);
diff --git a/e2fsck/util.c b/e2fsck/util.c
index db6a1cc..300993d 100644
--- a/e2fsck/util.c
+++ b/e2fsck/util.c
@@ -116,7 +116,7 @@ void log_err(e2fsck_t ctx, const char *fmt, ...)
 	}
 }
 
-void *e2fsck_allocate_memory(e2fsck_t ctx, unsigned int size,
+void *e2fsck_allocate_memory(e2fsck_t ctx, unsigned long size,
 			     const char *description)
 {
 	void *ret;
@@ -125,13 +125,12 @@ void *e2fsck_allocate_memory(e2fsck_t ctx, unsigned int size,
 #ifdef DEBUG_ALLOCATE_MEMORY
 	printf("Allocating %u bytes for %s...\n", size, description);
 #endif
-	ret = malloc(size);
-	if (!ret) {
+	if (ext2fs_get_memzero(size, &ret)) {
 		sprintf(buf, "Can't allocate %u bytes for %s\n",
 			size, description);
 		fatal_error(ctx, buf);
 	}
-	memset(ret, 0, size);
+
 	return ret;
 }
 
-- 
1.8.0

[PATCH 2/9] e2fsck: use proper types for variables

[Andreas Dilger]

Use ext2_ino_t instead of ino_t for referencing inode numbers.
Use loff_t for for file offsets, and dgrp_t for group numbers.

Cast products to ssize_t before multiplication to avoid overflow.

Signed-off-by: Andreas Dilger <adilger@dilger.ca>
Reviewed-by: Shilong Wang <wshilong@ddn.com>
Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197
---
 e2fsck/dx_dirinfo.c |  6 +++---
 e2fsck/e2fsck.h     | 11 ++++++-----
 e2fsck/rehash.c     |  2 +-
 e2fsck/super.c      |  2 +-
 lib/ext2fs/imager.c | 34 +++++++++++++++++++---------------
 misc/create_inode.c |  2 +-
 misc/dumpe2fs.c     |  2 +-
 misc/e2fuzz.c       |  8 ++++----
 misc/e2image.c      |  2 +-
 misc/tune2fs.c      |  2 +-
 10 files changed, 38 insertions(+), 33 deletions(-)

diff --git a/e2fsck/dx_dirinfo.c b/e2fsck/dx_dirinfo.c
index 89672b7..f0f6084 100644
--- a/e2fsck/dx_dirinfo.c
+++ b/e2fsck/dx_dirinfo.c
@@ -138,7 +138,7 @@ void e2fsck_free_dx_dir_info(e2fsck_t ctx)
 /*
  * Return the count of number of directories in the dx_dir_info structure
  */
-int e2fsck_get_num_dx_dirinfo(e2fsck_t ctx)
+ext2_ino_t e2fsck_get_num_dx_dirinfo(e2fsck_t ctx)
 {
 	return ctx->dx_dir_info_count;
 }
@@ -146,10 +146,10 @@ int e2fsck_get_num_dx_dirinfo(e2fsck_t ctx)
 /*
  * A simple interator function
  */
-struct dx_dir_info *e2fsck_dx_dir_info_iter(e2fsck_t ctx, int *control)
+struct dx_dir_info *e2fsck_dx_dir_info_iter(e2fsck_t ctx, ext2_ino_t *control)
 {
 	if (*control >= ctx->dx_dir_info_count)
 		return 0;
 
-	return(ctx->dx_dir_info + (*control)++);
+	return ctx->dx_dir_info + (*control)++;
 }
diff --git a/e2fsck/e2fsck.h b/e2fsck/e2fsck.h
index 253f8b5..5e7db42 100644
--- a/e2fsck/e2fsck.h
+++ b/e2fsck/e2fsck.h
@@ -235,12 +235,12 @@ struct e2fsck_struct {
 	char	*problem_log_fn;
 	int	flags;		/* E2fsck internal flags */
 	int	options;
-	int	blocksize;	/* blocksize */
+	unsigned blocksize;	/* blocksize */
 	blk64_t	use_superblock;	/* sb requested by user */
 	blk64_t	superblock;	/* sb used to open fs */
 	blk64_t	num_blocks;	/* Total number of blocks */
-	blk64_t free_blocks;
-	ino_t	free_inodes;
+	blk64_t	free_blocks;
+	ext2_ino_t free_inodes;
 	int	mount_flags;
 	int	openfs_flags;
 	blkid_cache blkid;	/* blkid cache */
@@ -478,8 +478,9 @@ extern void e2fsck_add_dx_dir(e2fsck_t ctx, ext2_ino_t ino,
 			      struct ext2_inode *inode, int num_blocks);
 extern struct dx_dir_info *e2fsck_get_dx_dir_info(e2fsck_t ctx, ext2_ino_t ino);
 extern void e2fsck_free_dx_dir_info(e2fsck_t ctx);
-extern int e2fsck_get_num_dx_dirinfo(e2fsck_t ctx);
-extern struct dx_dir_info *e2fsck_dx_dir_info_iter(e2fsck_t ctx, int *control);
+extern ext2_ino_t e2fsck_get_num_dx_dirinfo(e2fsck_t ctx);
+extern struct dx_dir_info *e2fsck_dx_dir_info_iter(e2fsck_t ctx,
+						   ext2_ino_t *control);
 
 /* ea_refcount.c */
 typedef __u64 ea_key_t;
diff --git a/e2fsck/rehash.c b/e2fsck/rehash.c
index a5fc1be..c9d667b 100644
--- a/e2fsck/rehash.c
+++ b/e2fsck/rehash.c
@@ -83,7 +83,7 @@ struct fill_dir_struct {
 	int max_array, num_array;
 	unsigned int dir_size;
 	int compress;
-	ino_t parent;
+	ext2_ino_t parent;
 	ext2_ino_t dir;
 };
 
diff --git a/e2fsck/super.c b/e2fsck/super.c
index e5932be..3aa1e87 100644
--- a/e2fsck/super.c
+++ b/e2fsck/super.c
@@ -595,7 +595,7 @@ void check_super_block(e2fsck_t ctx)
 	blk64_t	should_be;
 	struct problem_context	pctx;
 	blk64_t	free_blocks = 0;
-	ino_t	free_inodes = 0;
+	ext2_ino_t free_inodes = 0;
 	int     csum_flag, clear_test_fs_flag;
 
 	inodes_per_block = EXT2_INODES_PER_BLOCK(fs->super);
diff --git a/lib/ext2fs/imager.c b/lib/ext2fs/imager.c
index 7fd06f7..fee27ac 100644
--- a/lib/ext2fs/imager.c
+++ b/lib/ext2fs/imager.c
@@ -61,19 +61,20 @@ static int check_zero_block(char *buf, int blocksize)
 
 errcode_t ext2fs_image_inode_write(ext2_filsys fs, int fd, int flags)
 {
-	unsigned int	group, left, c, d;
+	dgrp_t		group;
+	ssize_t		left, c, d;
 	char		*buf, *cp;
 	blk64_t		blk;
 	ssize_t		actual;
 	errcode_t	retval;
-	off_t		r;
+	loff_t		r;
 
 	buf = malloc(fs->blocksize * BUF_BLOCKS);
 	if (!buf)
 		return ENOMEM;
 
 	for (group = 0; group < fs->group_desc_count; group++) {
-		blk = ext2fs_inode_table_loc(fs, (unsigned)group);
+		blk = ext2fs_inode_table_loc(fs, group);
 		if (!blk) {
 			retval = EXT2_ET_MISSING_INODE_TABLE;
 			goto errout;
@@ -107,23 +108,25 @@ errcode_t ext2fs_image_inode_write(ext2_filsys fs, int fd, int flags)
 					continue;
 				}
 				/* Find non-zero blocks */
-				for (d=1; d < c; d++) {
-					if (check_zero_block(cp + d*fs->blocksize, fs->blocksize))
+				for (d = 1; d < c; d++) {
+					if (check_zero_block(cp +
+							     d * fs->blocksize,
+							     fs->blocksize))
 						break;
 				}
 			skip_sparse:
-				actual = write(fd, cp, fs->blocksize * d);
+				actual = write(fd, cp, d * fs->blocksize);
 				if (actual == -1) {
 					retval = errno;
 					goto errout;
 				}
-				if (actual != (ssize_t) (fs->blocksize * d)) {
+				if (actual != d * fs->blocksize) {
 					retval = EXT2_ET_SHORT_WRITE;
 					goto errout;
 				}
 				blk += d;
 				left -= d;
-				cp += fs->blocksize * d;
+				cp += d * fs->blocksize;
 				c -= d;
 			}
 		}
@@ -141,7 +144,8 @@ errout:
 errcode_t ext2fs_image_inode_read(ext2_filsys fs, int fd,
 				  int flags EXT2FS_ATTR((unused)))
 {
-	unsigned int	group, c, left;
+	dgrp_t		group;
+	ssize_t		c, left;
 	char		*buf;
 	blk64_t		blk;
 	ssize_t		actual;
@@ -152,7 +156,7 @@ errcode_t ext2fs_image_inode_read(ext2_filsys fs, int fd,
 		return ENOMEM;
 
 	for (group = 0; group < fs->group_desc_count; group++) {
-		blk = ext2fs_inode_table_loc(fs, (unsigned)group);
+		blk = ext2fs_inode_table_loc(fs, group);
 		if (!blk) {
 			retval = EXT2_ET_MISSING_INODE_TABLE;
 			goto errout;
@@ -167,7 +171,7 @@ errcode_t ext2fs_image_inode_read(ext2_filsys fs, int fd,
 				retval = errno;
 				goto errout;
 			}
-			if (actual != (ssize_t) (fs->blocksize * c)) {
+			if (actual != fs->blocksize * c) {
 				retval = EXT2_ET_SHORT_READ;
 				goto errout;
 			}
@@ -249,7 +253,7 @@ errcode_t ext2fs_image_super_write(ext2_filsys fs, int fd,
 	}
 #endif
 
-	actual = write(fd, cp, fs->blocksize * fs->desc_blocks);
+	actual = write(fd, cp, (ssize_t)fs->blocksize * fs->desc_blocks);
 
 
 #ifdef WORDS_BIGENDIAN
@@ -265,7 +269,7 @@ errcode_t ext2fs_image_super_write(ext2_filsys fs, int fd,
 		retval = errno;
 		goto errout;
 	}
-	if (actual != (ssize_t) (fs->blocksize * fs->desc_blocks)) {
+	if (actual != (ssize_t)fs->blocksize * fs->desc_blocks) {
 		retval = EXT2_ET_SHORT_WRITE;
 		goto errout;
 	}
@@ -287,7 +291,7 @@ errcode_t ext2fs_image_super_read(ext2_filsys fs, int fd,
 	ssize_t		actual, size;
 	errcode_t	retval;
 
-	size = fs->blocksize * (fs->group_desc_count + 1);
+	size = (ssize_t)fs->blocksize * (fs->group_desc_count + 1);
 	buf = malloc(size);
 	if (!buf)
 		return ENOMEM;
@@ -311,7 +315,7 @@ errcode_t ext2fs_image_super_read(ext2_filsys fs, int fd,
 	memcpy(fs->super, buf, SUPERBLOCK_SIZE);
 
 	memcpy(fs->group_desc, buf + fs->blocksize,
-	       fs->blocksize * fs->group_desc_count);
+	       (ssize_t)fs->blocksize * fs->group_desc_count);
 
 	retval = 0;
 
diff --git a/misc/create_inode.c b/misc/create_inode.c
index 0091b72..6387425 100644
--- a/misc/create_inode.c
+++ b/misc/create_inode.c
@@ -599,7 +599,7 @@ out:
 	return err;
 }
 
-static int is_hardlink(struct hdlinks_s *hdlinks, dev_t dev, ino_t ino)
+static int is_hardlink(struct hdlinks_s *hdlinks, dev_t dev, ext2_ino_t ino)
 {
 	int i;
 
diff --git a/misc/dumpe2fs.c b/misc/dumpe2fs.c
index 384ce92..b812c93 100644
--- a/misc/dumpe2fs.c
+++ b/misc/dumpe2fs.c
@@ -362,7 +362,7 @@ static void print_inline_journal_information(ext2_filsys fs)
 	struct ext2_inode	inode;
 	ext2_file_t		journal_file;
 	errcode_t		retval;
-	ino_t			ino = fs->super->s_journal_inum;
+	ext2_ino_t		ino = fs->super->s_journal_inum;
 	char			buf[1024];
 
 	if (fs->flags & EXT2_FLAG_IMAGE_FILE)
diff --git a/misc/e2fuzz.c b/misc/e2fuzz.c
index 8576d4e..7c0f776 100644
--- a/misc/e2fuzz.c
+++ b/misc/e2fuzz.c
@@ -181,9 +181,9 @@ static int process_fs(const char *fsname)
 	int flags, fd;
 	ext2_filsys fs = NULL;
 	ext2fs_block_bitmap corrupt_map;
-	off_t hsize, count, off, offset, corrupt_bytes;
+	loff_t hsize, count, off, offset, corrupt_bytes;
 	unsigned char c;
-	off_t i;
+	loff_t i;
 
 	/* If mounted rw, force dryrun mode */
 	ret = ext2fs_check_if_mounted(fsname, &flags);
@@ -277,8 +277,8 @@ static int process_fs(const char *fsname)
 			c |= 0x80;
 		if (verbose)
 			printf("Corrupting byte %lld in block %lld to 0x%x\n",
-			       (long long) off % fs->blocksize,
-			       (long long) off / fs->blocksize, c);
+			       off % fs->blocksize,
+			       off / fs->blocksize, c);
 		if (dryrun)
 			continue;
 #ifdef HAVE_PWRITE64
diff --git a/misc/e2image.c b/misc/e2image.c
index 30f2543..2c0e14d 100644
--- a/misc/e2image.c
+++ b/misc/e2image.c
@@ -312,7 +312,7 @@ struct process_block_struct {
  * structure, so there's no point in letting the ext2fs library read
  * the inode again.
  */
-static ino_t stashed_ino = 0;
+static ext2_ino_t stashed_ino = 0;
 static struct ext2_inode *stashed_inode;
 
 static errcode_t meta_get_blocks(ext2_filsys fs EXT2FS_ATTR((unused)),
diff --git a/misc/tune2fs.c b/misc/tune2fs.c
index 7d2d38d..0324cbb 100644
--- a/misc/tune2fs.c
+++ b/misc/tune2fs.c
@@ -365,7 +365,7 @@ static errcode_t remove_journal_inode(ext2_filsys fs)
 {
 	struct ext2_inode	inode;
 	errcode_t		retval;
-	ino_t			ino = fs->super->s_journal_inum;
+	ext2_ino_t		ino = fs->super->s_journal_inum;
 
 	retval = ext2fs_read_inode(fs, ino,  &inode);
 	if (retval) {
-- 
1.8.0

[PATCH 3/9] e2fsck: avoid mallinfo() if over 2GB allocated

[Andreas Dilger]

Don't use mallinfo() for determining the amount of memory used if it
is over 2GB.  Otherwise, the signed ints used by this interface can
can overflow and return garbage values.  This makes the actual amount
of memory used by e2fsck misleading and hard to determine.

Instead, use brk() to get the total amount of memory allocated, and print
this if the more detailed mallinfo() information is not suitable for use.

There does not appear to be a mallinfo64() variant of this function.
There does appear to be an abomination named malloc_info() that writes
XML-formatted malloc stats to a FILE stream that would need to be read
and parsed in order to get these stats, but that doesn't seem worthwhile.

Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Reviewed-by: Shilong Wang <wshilong@ddn.com>
Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197
---
 e2fsck/scantest.c |  4 ++--
 e2fsck/util.c     | 26 +++++++++++++-------------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/e2fsck/scantest.c b/e2fsck/scantest.c
index 6131141..ed3595f 100644
--- a/e2fsck/scantest.c
+++ b/e2fsck/scantest.c
@@ -76,8 +76,8 @@ static void print_resource_track(struct resource_track *track)
 	gettimeofday(&time_end, 0);
 	getrusage(RUSAGE_SELF, &r);
 
-	printf(_("Memory used: %d, elapsed time: %6.3f/%6.3f/%6.3f\n"),
-	       (int) (((char *) sbrk(0)) - ((char *) track->brk_start)),
+	printf(_("Memory used: %lu, elapsed time: %6.3f/%6.3f/%6.3f\n"),
+	       (unsigned long)((char *)sbrk(0) - (char *)track->brk_start),
 	       timeval_subtract(&time_end, &track->time_start),
 	       timeval_subtract(&r.ru_utime, &track->user_start),
 	       timeval_subtract(&r.ru_stime, &track->system_start));
diff --git a/e2fsck/util.c b/e2fsck/util.c
index 300993d..e3d92b3 100644
--- a/e2fsck/util.c
+++ b/e2fsck/util.c
@@ -421,9 +421,6 @@ void print_resource_track(e2fsck_t ctx, const char *desc,
 #ifdef HAVE_GETRUSAGE
 	struct rusage r;
 #endif
-#ifdef HAVE_MALLINFO
-	struct mallinfo	malloc_info;
-#endif
 	struct timeval time_end;
 
 	if ((desc && !(ctx->options & E2F_OPT_TIME2)) ||
@@ -436,18 +433,21 @@ void print_resource_track(e2fsck_t ctx, const char *desc,
 	if (desc)
 		log_out(ctx, "%s: ", desc);
 
+#define kbytes(x)	(((unsigned long long)(x) + 1023) / 1024)
 #ifdef HAVE_MALLINFO
-#define kbytes(x)	(((unsigned long)(x) + 1023) / 1024)
-
-	malloc_info = mallinfo();
-	log_out(ctx, _("Memory used: %luk/%luk (%luk/%luk), "),
-		kbytes(malloc_info.arena), kbytes(malloc_info.hblkhd),
-		kbytes(malloc_info.uordblks), kbytes(malloc_info.fordblks));
-#else
-	log_out(ctx, _("Memory used: %lu, "),
-		(unsigned long) (((char *) sbrk(0)) -
-				 ((char *) track->brk_start)));
+	/* don't use mallinfo() if over 2GB used, since it returns "int" */
+	if ((char *)sbrk(0) - (char *)track->brk_start < 2ULL << 30) {
+		struct mallinfo	malloc_info = mallinfo();
+
+		log_out(ctx, _("Memory used: %lluk/%lluk (%lluk/%lluk), "),
+			kbytes(malloc_info.arena), kbytes(malloc_info.hblkhd),
+			kbytes(malloc_info.uordblks),
+			kbytes(malloc_info.fordblks));
+	} else
 #endif
+	log_out(ctx, _("Memory used: %lluk, "),
+		kbytes(((char *)sbrk(0)) - ((char *)track->brk_start)));
+
 #ifdef HAVE_GETRUSAGE
 	getrusage(RUSAGE_SELF, &r);
 
-- 
1.8.0

[PATCH 4/9] e2fsck: reduce memory usage for many directories

[Andreas Dilger]

Pack struct dx_dir_info and dx_dirblock_info properly in memory, to
avoid holes, and fields are not larger than necessary.  This reduces
the memory needed for each hashed dir, according to pahole(1) from:

    struct dx_dir_info {
        /* size: 32, cachelines: 1, members: 6 */
        /* sum members: 26, holes: 1, sum holes: 2 */
        /* padding: 4 */
    };
    struct dx_dirblock_info {
        /* size: 56, cachelines: 1, members: 9 */
        /* sum members: 48, holes: 2, sum holes: 8 */
        /* last cacheline: 56 bytes */
    };

to 8 bytes less for each directory and directory block, and leaves
space for future use if needed (e.g. larger numblocks):

    struct dx_dir_info {
        /* size: 24, cachelines: 1, members: 6 */
        /* sum members: 20, holes: 1, sum holes: 4 */
        /* bit holes: 1, sum bit holes: 7 bits */
    };
    struct dx_dirblock_info {
        /* size: 48, cachelines: 1, members: 9 */
    };

Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197
---
 e2fsck/dx_dirinfo.c |  3 +--
 e2fsck/e2fsck.h     | 14 +++++++-------
 e2fsck/pass2.c      | 12 ++++++------
 3 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/e2fsck/dx_dirinfo.c b/e2fsck/dx_dirinfo.c
index f0f6084..caca3e3 100644
--- a/e2fsck/dx_dirinfo.c
+++ b/e2fsck/dx_dirinfo.c
@@ -73,11 +73,10 @@ void e2fsck_add_dx_dir(e2fsck_t ctx, ext2_ino_t ino, struct ext2_inode *inode,
 	dir->ino = ino;
 	dir->numblocks = num_blocks;
 	dir->hashversion = 0;
-	dir->casefolded_hash = inode->i_flags & EXT4_CASEFOLD_FL;
+	dir->casefolded_hash = !!(inode->i_flags & EXT4_CASEFOLD_FL);
 	dir->dx_block = e2fsck_allocate_memory(ctx, num_blocks
 				       * sizeof (struct dx_dirblock_info),
 				       "dx_block info array");
-
 }
 
 /*
diff --git a/e2fsck/e2fsck.h b/e2fsck/e2fsck.h
index 5e7db42..feb605c 100644
--- a/e2fsck/e2fsck.h
+++ b/e2fsck/e2fsck.h
@@ -104,12 +104,12 @@ struct dir_info {
  * directories which contain a hash tree index.
  */
 struct dx_dir_info {
-	ext2_ino_t		ino; 		/* Inode number */
-	int			numblocks;	/* number of blocks */
-	int			hashversion;
-	short			depth;		/* depth of tree */
-	struct dx_dirblock_info	*dx_block; 	/* Array of size numblocks */
-	int			casefolded_hash;
+	ext2_ino_t		ino;		/* Inode number */
+	short			depth;		/* depth of tree (15 bits) */
+	__u8			hashversion;
+	__u8			casefolded_hash:1;
+	blk_t			numblocks;	/* number of blocks in dir */
+	struct dx_dirblock_info	*dx_block;	/* Array of size numblocks */
 };
 
 #define DX_DIRBLOCK_ROOT	1
@@ -120,8 +120,8 @@ struct dx_dir_info {
 
 struct dx_dirblock_info {
 	int		type;
-	blk64_t		phys;
 	int		flags;
+	blk64_t		phys;
 	blk64_t		parent;
 	blk64_t		previous;
 	ext2_dirhash_t	min_hash;
diff --git a/e2fsck/pass2.c b/e2fsck/pass2.c
index 5c3f7b8..0fa6233 100644
--- a/e2fsck/pass2.c
+++ b/e2fsck/pass2.c
@@ -71,8 +71,8 @@ static int allocate_dir_block(e2fsck_t ctx,
 			      struct ext2_db_entry2 *dir_blocks_info,
 			      char *buf, struct problem_context *pctx);
 static void clear_htree(e2fsck_t ctx, ext2_ino_t ino);
-static int htree_depth(struct dx_dir_info *dx_dir,
-		       struct dx_dirblock_info *dx_db);
+static short htree_depth(struct dx_dir_info *dx_dir,
+			 struct dx_dirblock_info *dx_db);
 static EXT2_QSORT_TYPE special_dir_block_cmp(const void *a, const void *b);
 
 struct check_dir_struct {
@@ -132,7 +132,7 @@ void e2fsck_pass2(e2fsck_t ctx)
 	struct dx_dirblock_info	*dx_db;
 	int			b;
 	ext2_ino_t		i;
-	int			depth;
+	short			depth;
 	problem_t		code;
 	int			bad_dir;
 	int (*check_dir_func)(ext2_filsys fs,
@@ -311,10 +311,10 @@ cleanup:
 }
 
 #define MAX_DEPTH 32000
-static int htree_depth(struct dx_dir_info *dx_dir,
-		       struct dx_dirblock_info *dx_db)
+static short htree_depth(struct dx_dir_info *dx_dir,
+			 struct dx_dirblock_info *dx_db)
 {
-	int	depth = 0;
+	short depth = 0;
 
 	while (dx_db->type != DX_DIRBLOCK_ROOT && depth < MAX_DEPTH) {
 		dx_db = &dx_dir->dx_block[dx_db->parent];
-- 
1.8.0

[PATCH 5/9] debugfs: allow comment lines in command file

[Andreas Dilger]

Allow comment lines with '#' at the start of the line in the command
file passed in to debugfs via the "-f" option or from standard input.

Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197
---
 debugfs/debugfs.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/debugfs/debugfs.c b/debugfs/debugfs.c
index 15b0121..60931a9 100644
--- a/debugfs/debugfs.c
+++ b/debugfs/debugfs.c
@@ -2486,6 +2486,10 @@ static int source_file(const char *cmd_file, int ss_idx)
 	while (!feof(f)) {
 		if (fgets(buf, sizeof(buf), f) == NULL)
 			break;
+		if (buf[0] == '#') {
+			printf("%s", buf);
+			continue;
+		}
 		cp = strchr(buf, '\n');
 		if (cp)
 			*cp = 0;
-- 
1.8.0

[PATCH 6/9] debugfs: print inode numbers as unsigned

[Andreas Dilger]

Print inode numbers as unsigned values, to avoid printing negative
numbers for inodes above 2B.

Flags should be printed as hex instead of signed decimal values.

Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197
---
 debugfs/debugfs.c      | 18 +++++++++---------
 debugfs/do_journal.c   |  4 ++--
 debugfs/extent_inode.c |  4 ++--
 debugfs/htree.c        |  2 +-
 4 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/debugfs/debugfs.c b/debugfs/debugfs.c
index 60931a9..08fb9a3 100644
--- a/debugfs/debugfs.c
+++ b/debugfs/debugfs.c
@@ -467,7 +467,7 @@ void do_show_super_stats(int argc, char *argv[],
 	}
 	for (i=0; i < current_fs->group_desc_count; i++)
 		numdirs += ext2fs_bg_used_dirs_count(current_fs, i);
-	fprintf(out, "Directories:              %d\n", numdirs);
+	fprintf(out, "Directories:              %u\n", numdirs);
 
 	if (header_only) {
 		close_pager(out);
@@ -718,7 +718,7 @@ static void dump_extents(FILE *f, const char *prefix, ext2_ino_t ino,
 				continue;
 			}
 
-			fprintf(f, "%s(ETB%d):%lld",
+			fprintf(f, "%s(ETB%d):%llu",
 				printed ? ", " : "", info.curr_level,
 				extent.e_pblk);
 			printed = 1;
@@ -851,10 +851,10 @@ void internal_dump_inode(FILE *out, const char *prefix,
 	if (LINUX_S_ISREG(inode->i_mode) || LINUX_S_ISDIR(inode->i_mode))
 		fprintf(out, "%llu\n", EXT2_I_SIZE(inode));
 	else
-		fprintf(out, "%d\n", inode->i_size);
+		fprintf(out, "%u\n", inode->i_size);
 	if (os == EXT2_OS_HURD)
 		fprintf(out,
-			"%sFile ACL: %d Translator: %d\n",
+			"%sFile ACL: %u Translator: %u\n",
 			prefix,
 			inode->i_file_acl,
 			inode->osd1.hurd1.h_i_translator);
@@ -864,13 +864,13 @@ void internal_dump_inode(FILE *out, const char *prefix,
 			inode->i_file_acl | ((long long)
 				(inode->osd2.linux2.l_i_file_acl_high) << 32));
 	if (os != EXT2_OS_HURD)
-		fprintf(out, "%sLinks: %d   Blockcount: %llu\n",
+		fprintf(out, "%sLinks: %u   Blockcount: %llu\n",
 			prefix, inode->i_links_count,
 			(((unsigned long long)
 			  inode->osd2.linux2.l_i_blocks_hi << 32)) +
 			inode->i_blocks);
 	else
-		fprintf(out, "%sLinks: %d   Blockcount: %u\n",
+		fprintf(out, "%sLinks: %u   Blockcount: %u\n",
 			prefix, inode->i_links_count, inode->i_blocks);
 	switch (os) {
 	    case EXT2_OS_HURD:
@@ -880,7 +880,7 @@ void internal_dump_inode(FILE *out, const char *prefix,
 	    default:
 		frag = fsize = 0;
 	}
-	fprintf(out, "%sFragment:  Address: %d    Number: %d    Size: %d\n",
+	fprintf(out, "%sFragment:  Address: %u    Number: %u    Size: %u\n",
 		prefix, inode->i_faddr, frag, fsize);
 	if (is_large_inode && large_inode->i_extra_isize >= 24) {
 		fprintf(out, "%s ctime: 0x%08x:%08x -- %s", prefix,
@@ -1397,7 +1397,7 @@ void do_modify_inode(int argc, char *argv[], int sci_idx EXT2FS_ATTR((unused)),
 		modify_u8(argv[0], "Fragment size", decimal_format, fsize);
 
 	for (i=0;  i < EXT2_NDIR_BLOCKS; i++) {
-		sprintf(buf, "Direct Block #%d", i);
+		sprintf(buf, "Direct Block #%u", i);
 		modify_u32(argv[0], buf, decimal_format, &inode.i_block[i]);
 	}
 	modify_u32(argv[0], "Indirect Block", decimal_format,
@@ -2133,7 +2133,7 @@ void do_imap(int argc, char *argv[], int sci_idx EXT2FS_ATTR((unused)),
 		block;
 	offset &= (EXT2_BLOCK_SIZE(current_fs->super) - 1);
 
-	printf("Inode %d is part of block group %lu\n"
+	printf("Inode %u is part of block group %lu\n"
 	       "\tlocated at block %lu, offset 0x%04lx\n", ino, group,
 	       block_nr, offset);
 
diff --git a/debugfs/do_journal.c b/debugfs/do_journal.c
index eeb363e..15ef682 100644
--- a/debugfs/do_journal.c
+++ b/debugfs/do_journal.c
@@ -59,7 +59,7 @@ static journal_t *current_journal = NULL;
 static void journal_dump_trans(journal_transaction_t *trans EXT2FS_ATTR((unused)),
 			       const char *tag EXT2FS_ATTR((unused)))
 {
-	dbg_printf("TRANS %p(%s): tid=%d start=%llu block=%llu end=%llu "
+	dbg_printf("TRANS %p(%s): tid=%u start=%llu block=%llu end=%llu "
 		   "flags=0x%x\n", trans, tag, trans->tid, trans->start,
 		   trans->block, trans->end, trans->flags);
 }
@@ -912,7 +912,7 @@ void do_journal_open(int argc, char *argv[], int sci_idx EXT2FS_ATTR((unused)),
 	}
 	journal = current_journal;
 
-	dbg_printf("JOURNAL: seq=%d tailseq=%d start=%lu first=%lu "
+	dbg_printf("JOURNAL: seq=%u tailseq=%u start=%lu first=%lu "
 		   "maxlen=%lu\n", journal->j_tail_sequence,
 		   journal->j_transaction_sequence, journal->j_tail,
 		   journal->j_first, journal->j_last);
diff --git a/debugfs/extent_inode.c b/debugfs/extent_inode.c
index ada1308..6706629 100644
--- a/debugfs/extent_inode.c
+++ b/debugfs/extent_inode.c
@@ -77,7 +77,7 @@ void do_extent_open(int argc, char *argv[], int sci_idx EXT2FS_ATTR((unused)),
 
 	if (argc == 1) {
 		if (current_ino)
-			printf("Current inode is %d\n", current_ino);
+			printf("Current inode is %u\n", current_ino);
 		else
 			printf("No current inode\n");
 		return;
@@ -107,7 +107,7 @@ void do_extent_open(int argc, char *argv[], int sci_idx EXT2FS_ATTR((unused)),
 	cp = strchr(extent_prompt, ':');
 	if (cp)
 		*cp = 0;
-	sprintf(extent_prompt + strlen(extent_prompt), " (extent ino %d): ",
+	sprintf(extent_prompt + strlen(extent_prompt), " (extent ino %u): ",
 		current_ino);
 	ss_add_request_table(sci_idx, &extent_cmds, 1, &ret);
 	ss_set_prompt(sci_idx, extent_prompt);
diff --git a/debugfs/htree.c b/debugfs/htree.c
index 3aae3c2..7fae7f1 100644
--- a/debugfs/htree.c
+++ b/debugfs/htree.c
@@ -288,7 +288,7 @@ void do_htree_dump(int argc, char *argv[], int sci_idx EXT2FS_ATTR((unused)),
 	fprintf(pager, "\t Hash Version: %d\n", rootnode->hash_version);
 	fprintf(pager, "\t Info length: %d\n", rootnode->info_length);
 	fprintf(pager, "\t Indirect levels: %d\n", rootnode->indirect_levels);
-	fprintf(pager, "\t Flags: %d\n", rootnode->unused_flags);
+	fprintf(pager, "\t Flags: %#x\n", rootnode->unused_flags);
 
 	ent = (struct ext2_dx_entry *)
 		((char *)rootnode + rootnode->info_length);
-- 
1.8.0

[PATCH 7/9] e2fsck: fix overflow if more than 4B inodes

[Andreas Dilger]

Even though we don't have support for filesystems with over 4B inodes
in the current e2fsprogs, this may happen in the future.  There are
latent overflow bugs when calculating the number of inodes in the
filesystem that can trivially be fixed now, rather than waiting for
them to be hit at some point in the future.  The block number calcs
are already correct in this code.

Signed-off-by: Andreas Dilger <adilger@dilger.ca>
Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197
---
 e2fsck/pass5.c       | 2 +-
 lib/ext2fs/bitmaps.c | 3 ++-
 lib/ext2fs/imager.c  | 6 ++++--
 misc/fuse2fs.c       | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/e2fsck/pass5.c b/e2fsck/pass5.c
index 3a5c88d..c1d45a5 100644
--- a/e2fsck/pass5.c
+++ b/e2fsck/pass5.c
@@ -842,7 +842,7 @@ static void check_inode_end(e2fsck_t ctx)
 
 	clear_problem_context(&pctx);
 
-	end = EXT2_INODES_PER_GROUP(fs->super) * fs->group_desc_count;
+	end = (__u64)EXT2_INODES_PER_GROUP(fs->super) * fs->group_desc_count;
 	pctx.errcode = ext2fs_fudge_inode_bitmap_end(fs->inode_map, end,
 						     &save_inodes_count);
 	if (pctx.errcode) {
diff --git a/lib/ext2fs/bitmaps.c b/lib/ext2fs/bitmaps.c
index e25db2c..834a396 100644
--- a/lib/ext2fs/bitmaps.c
+++ b/lib/ext2fs/bitmaps.c
@@ -62,7 +62,8 @@ errcode_t ext2fs_allocate_inode_bitmap(ext2_filsys fs,
 
 	start = 1;
 	end = fs->super->s_inodes_count;
-	real_end = (EXT2_INODES_PER_GROUP(fs->super) * fs->group_desc_count);
+	real_end = (__u64)EXT2_INODES_PER_GROUP(fs->super) *
+		fs->group_desc_count;
 
 	/* Are we permitted to use new-style bitmaps? */
 	if (fs->flags & EXT2_FLAG_64BITS)
diff --git a/lib/ext2fs/imager.c b/lib/ext2fs/imager.c
index fee27ac..09cd508 100644
--- a/lib/ext2fs/imager.c
+++ b/lib/ext2fs/imager.c
@@ -344,7 +344,8 @@ errcode_t ext2fs_image_bitmap_write(ext2_filsys fs, int fd, int flags)
 		}
 		bmap = fs->inode_map;
 		itr = 1;
-		cnt = EXT2_INODES_PER_GROUP(fs->super) * fs->group_desc_count;
+		cnt = (__u64)EXT2_INODES_PER_GROUP(fs->super) *
+			fs->group_desc_count;
 		size = (EXT2_INODES_PER_GROUP(fs->super) / 8);
 	} else {
 		if (!fs->block_map) {
@@ -419,7 +420,8 @@ errcode_t ext2fs_image_bitmap_read(ext2_filsys fs, int fd, int flags)
 		}
 		bmap = fs->inode_map;
 		itr = 1;
-		cnt = EXT2_INODES_PER_GROUP(fs->super) * fs->group_desc_count;
+		cnt = (__u64)EXT2_INODES_PER_GROUP(fs->super) *
+			fs->group_desc_count;
 		size = (EXT2_INODES_PER_GROUP(fs->super) / 8);
 	} else {
 		if (!fs->block_map) {
diff --git a/misc/fuse2fs.c b/misc/fuse2fs.c
index be2cd1d..31493e2 100644
--- a/misc/fuse2fs.c
+++ b/misc/fuse2fs.c
@@ -2364,7 +2364,7 @@ static int op_statfs(const char *path EXT2FS_ATTR((unused)),
 		overhead = 0;
 	else
 		overhead = fs->desc_blocks +
-			   fs->group_desc_count *
+			   (blk64_t)fs->group_desc_count *
 			   (fs->inode_blocks_per_group + 2);
 	reserved = ext2fs_r_blocks_count(fs->super);
 	if (!reserved)
-- 
1.8.0

[PATCH 8/9] e2fsck: consistently use ext2fs_get_mem()

[Andreas Dilger]

Consistently use ext2fs_get_mem() and ext2fs_free_mem() instead of
calling malloc() and free() directly in e2fsck.  In several places
it is possible to use ext2fs_get_memzero() instead of explicitly
calling memset() on the memory afterward.

This is just a code cleanup, and does not fix any specific bugs.

Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197
---
 e2fsck/ea_refcount.c |  6 ++----
 e2fsck/emptydir.c    | 14 +++++++-------
 e2fsck/extend.c      |  9 +++++----
 e2fsck/extents.c     | 16 ++++++++--------
 e2fsck/pass1b.c      | 12 ++++++------
 e2fsck/region.c      | 18 ++++++++++--------
 6 files changed, 38 insertions(+), 37 deletions(-)

diff --git a/e2fsck/ea_refcount.c b/e2fsck/ea_refcount.c
index ecb1986..aa5d7d7 100644
--- a/e2fsck/ea_refcount.c
+++ b/e2fsck/ea_refcount.c
@@ -53,10 +53,9 @@ errcode_t ea_refcount_create(size_t size, ext2_refcount_t *ret)
 	errcode_t	retval;
 	size_t		bytes;
 
-	retval = ext2fs_get_mem(sizeof(struct ea_refcount), &refcount);
+	retval = ext2fs_get_memzero(sizeof(struct ea_refcount), &refcount);
 	if (retval)
 		return retval;
-	memset(refcount, 0, sizeof(struct ea_refcount));
 
 	if (!size)
 		size = 500;
@@ -66,10 +65,9 @@ errcode_t ea_refcount_create(size_t size, ext2_refcount_t *ret)
 	printf("Refcount allocated %zu entries, %zu bytes.\n",
 	       refcount->size, bytes);
 #endif
-	retval = ext2fs_get_mem(bytes, &refcount->list);
+	retval = ext2fs_get_memzero(bytes, &refcount->list);
 	if (retval)
 		goto errout;
-	memset(refcount->list, 0, bytes);
 
 	refcount->count = 0;
 	refcount->cursor = 0;
diff --git a/e2fsck/emptydir.c b/e2fsck/emptydir.c
index a3bfd46..7aea7b6 100644
--- a/e2fsck/emptydir.c
+++ b/e2fsck/emptydir.c
@@ -44,12 +44,11 @@ empty_dir_info init_empty_dir(e2fsck_t ctx)
 	empty_dir_info	edi;
 	errcode_t	retval;
 
-	edi = malloc(sizeof(struct empty_dir_info_struct));
-	if (!edi)
+	edi = e2fsck_allocate_memzero(ctx, sizeof(struct empty_dir_info_struct),
+				      "empty dir info");
+	if (retval)
 		return NULL;
 
-	memset(edi, 0, sizeof(struct empty_dir_info_struct));
-
 	retval = ext2fs_init_dblist(ctx->fs, &edi->empty_dblist);
 	if (retval)
 		goto errout;
@@ -83,7 +82,7 @@ void free_empty_dirblock(empty_dir_info edi)
 		ext2fs_free_inode_bitmap(edi->dir_map);
 
 	memset(edi, 0, sizeof(struct empty_dir_info_struct));
-	free(edi);
+	ext2fs_free_mem(&edi);
 }
 
 void add_empty_dirblock(empty_dir_info edi,
@@ -182,13 +181,14 @@ void process_empty_dirblock(e2fsck_t ctx, empty_dir_info edi)
 	if (!edi)
 		return;
 
-	edi->block_buf = malloc(ctx->fs->blocksize * 3);
+	retval = ext2f_get_mem(ctx, ctx->fs->blocksize * 3,
+			       &edi->block_buf);
 
 	if (edi->block_buf) {
 		(void) ext2fs_dblist_iterate2(edi->empty_dblist,
 					      fix_directory, &edi);
 	}
-	free(edi->block_buf);
+	ext2fs_free_mem(&edi->block_buf);
 	free_empty_dirblock(edi);
 }
 
diff --git a/e2fsck/extend.c b/e2fsck/extend.c
index bdb62c3..9d17e44 100644
--- a/e2fsck/extend.c
+++ b/e2fsck/extend.c
@@ -31,6 +31,7 @@ int main(int argc, char **argv)
 	int	nblocks, blocksize;
 	int	fd;
 	char	*block;
+	errcode_t retval;
 	int	ret;
 
 	if (argc != 4)
@@ -45,13 +46,12 @@ int main(int argc, char **argv)
 		exit(1);
 	}
 
-	block = malloc(blocksize);
-	if (block == 0) {
+	retval = ext2fs_get_memzero(blocksize, &block);
+	if (retval) {
 		fprintf(stderr, _("Couldn't allocate block buffer (size=%d)\n"),
 			blocksize);
 		exit(1);
 	}
-	memset(block, 0, blocksize);
 
 	fd = open(filename, O_RDWR);
 	if (fd < 0) {
@@ -78,5 +78,6 @@ int main(int argc, char **argv)
 		perror("read");
 		exit(1);
 	}
-	exit(0);
+	ext2fs_free_mem(&block);
+	return(0);
 }
diff --git a/e2fsck/extents.c b/e2fsck/extents.c
index 3073725..2c91c8c 100644
--- a/e2fsck/extents.c
+++ b/e2fsck/extents.c
@@ -327,7 +327,7 @@ err:
 /* Rebuild the extents immediately */
 static errcode_t e2fsck_rebuild_extents(e2fsck_t ctx, ext2_ino_t ino)
 {
-	struct extent_list	list;
+	struct extent_list list = { 0 };
 	errcode_t err;
 
 	if (!ext2fs_has_feature_extents(ctx->fs->super) ||
@@ -336,9 +336,8 @@ static errcode_t e2fsck_rebuild_extents(e2fsck_t ctx, ext2_ino_t ino)
 		return 0;
 
 	e2fsck_read_bitmaps(ctx);
-	memset(&list, 0, sizeof(list));
-	err = ext2fs_get_mem(sizeof(struct ext2fs_extent) * NUM_EXTENTS,
-				&list.extents);
+	err = ext2fs_get_array(NUM_EXTENTS, sizeof(struct ext2fs_extent),
+			       &list.extents);
 	if (err)
 		return err;
 	list.size = NUM_EXTENTS;
@@ -354,7 +353,7 @@ static void rebuild_extents(e2fsck_t ctx, const char *pass_name, int pr_header)
 #ifdef RESOURCE_TRACK
 	struct resource_track	rtrack;
 #endif
-	struct extent_list	list;
+	struct extent_list	list = { 0 };
 	int			first = 1;
 	ext2_ino_t		ino = 0;
 	errcode_t		retval;
@@ -374,10 +373,11 @@ static void rebuild_extents(e2fsck_t ctx, const char *pass_name, int pr_header)
 	clear_problem_context(&pctx);
 	e2fsck_read_bitmaps(ctx);
 
-	memset(&list, 0, sizeof(list));
-	retval = ext2fs_get_mem(sizeof(struct ext2fs_extent) * NUM_EXTENTS,
-				&list.extents);
 	list.size = NUM_EXTENTS;
+	retval = ext2fs_get_array(sizeof(struct ext2fs_extent),
+				  list.size, &list.extents);
+	if (retval)
+		return;
 	while (1) {
 		retval = ext2fs_find_first_set_inode_bitmap2(
 				ctx->inodes_to_rebuild, ino + 1,
diff --git a/e2fsck/pass1b.c b/e2fsck/pass1b.c
index 5693b9c..5a6007d 100644
--- a/e2fsck/pass1b.c
+++ b/e2fsck/pass1b.c
@@ -180,10 +180,10 @@ static void inode_dnode_free(dnode_t *node,
 	di = (struct dup_inode *) dnode_get(node);
 	for (p = di->cluster_list; p; p = next) {
 		next = p->next;
-		free(p);
+		ext2fs_free_mem(&p);
 	}
-	free(di);
-	free(node);
+	ext2fs_free_mem(&di);
+	ext2fs_free_mem(&node);
 }
 
 /*
@@ -198,10 +198,10 @@ static void cluster_dnode_free(dnode_t *node,
 	dc = (struct dup_cluster *) dnode_get(node);
 	for (p = dc->inode_list; p; p = next) {
 		next = p->next;
-		free(p);
+		ext2fs_free_mem(&p);
 	}
-	free(dc);
-	free(node);
+	ext2fs_free_mem(&dc);
+	ext2fs_free_mem(&node);
 }
 
 
diff --git a/e2fsck/region.c b/e2fsck/region.c
index d5b37df..788e0d0 100644
--- a/e2fsck/region.c
+++ b/e2fsck/region.c
@@ -36,11 +36,12 @@ struct region_struct {
 region_t region_create(region_addr_t min, region_addr_t max)
 {
 	region_t	region;
+	errcode_t	retval;
 
-	region = malloc(sizeof(struct region_struct));
-	if (!region)
+	retval = ext2fs_get_memzero(sizeof(struct region_struct), &region);
+	if (retval)
 		return NULL;
-	memset(region, 0, sizeof(struct region_struct));
+
 	region->min = min;
 	region->max = max;
 	region->last = NULL;
@@ -53,16 +54,17 @@ void region_free(region_t region)
 
 	for (r = region->allocated; r; r = next) {
 		next = r->next;
-		free(r);
+		ext2fs_free_mem(&r);
 	}
 	memset(region, 0, sizeof(struct region_struct));
-	free(region);
+	ext2fs_free_mem(&region);
 }
 
 int region_allocate(region_t region, region_addr_t start, int n)
 {
 	struct region_el	*r, *new_region, *prev, *next;
 	region_addr_t end;
+	errcode_t retval;
 
 	end = start+n;
 	if ((start < region->min) || (end > region->max))
@@ -105,7 +107,7 @@ int region_allocate(region_t region, region_addr_t start, int n)
 				if (end == next->start) {
 					r->end = next->end;
 					r->next = next->next;
-					free(next);
+					ext2fs_free_mem(&next);
 					if (!r->next)
 						region->last = r;
 					return 0;
@@ -121,8 +123,8 @@ int region_allocate(region_t region, region_addr_t start, int n)
 	 * Insert a new region element structure into the linked list
 	 */
 append_to_list:
-	new_region = malloc(sizeof(struct region_el));
-	if (!new_region)
+	retval = ext2fs_get_mem(sizeof(struct region_el), &new_region);
+	if (retval)
 		return -1;
 	new_region->start = start;
 	new_region->end = start + n;
-- 
1.8.0

[PATCH 9/9] misc: handle very large files with filefrag

[Andreas Dilger]

Avoid overflowing the column-width calc printing files over 4B blocks.

Document the [KMG] suffixes for the "-b <blocksize>" option.

The blocksize is limited to at most 1GiB blocksize to avoid shifting
all extents down to zero GB in size.  Even the use of 1GB blocksize
is unlikely, but non-ext4 filesystems may use multi-GB extents.

Signed-off-by: Andreas Dilger <adilger@dilger.ca>
Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197
---
 misc/filefrag.8.in |  4 ++--
 misc/filefrag.c    | 36 ++++++++++++++++++++++++------------
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/misc/filefrag.8.in b/misc/filefrag.8.in
index 5292672..4b89e72 100644
--- a/misc/filefrag.8.in
+++ b/misc/filefrag.8.in
@@ -33,8 +33,8 @@ testing purposes.
 .BI \-b blocksize
 Use
 .I blocksize
-in bytes for output instead of the filesystem blocksize.
-For compatibility with earlier versions of
+in bytes, or with [KMG] suffix, up to 1GB for output instead of the
+filesystem blocksize.  For compatibility with earlier versions of
 .BR filefrag ,
 if
 .I blocksize
diff --git a/misc/filefrag.c b/misc/filefrag.c
index 1eec146..032535f 100644
--- a/misc/filefrag.c
+++ b/misc/filefrag.c
@@ -53,7 +53,7 @@ extern int optind;
 #include <ext2fs/fiemap.h>
 
 int verbose = 0;
-int blocksize;		/* Use specified blocksize (default 1kB) */
+unsigned int blocksize;	/* Use specified blocksize (default 1kB) */
 int sync_file = 0;	/* fsync file before getting the mapping */
 int xattr_map = 0;	/* get xattr mapping */
 int force_bmap;	/* force use of FIBMAP instead of FIEMAP */
@@ -73,7 +73,7 @@ const char *hex_fmt = "%4d: %*llx..%*llx: %*llx..%*llx: %6llx: %s\n";
 #define	EXT4_EXTENTS_FL			0x00080000 /* Inode uses extents */
 #define	EXT3_IOC_GETFLAGS		_IOR('f', 1, long)
 
-static int int_log2(int arg)
+static int ulong_log2(unsigned long arg)
 {
 	int     l = 0;
 
@@ -85,7 +85,7 @@ static int int_log2(int arg)
 	return l;
 }
 
-static int int_log10(unsigned long long arg)
+static int ulong_log10(unsigned long long arg)
 {
 	int     l = 0;
 
@@ -452,17 +452,17 @@ static int frag_report(const char *filename)
 	}
 	last_device = st.st_dev;
 
-	width = int_log10(fsinfo.f_blocks);
+	width = ulong_log10(fsinfo.f_blocks);
 	if (width > physical_width)
 		physical_width = width;
 
 	numblocks = (st.st_size + blksize - 1) / blksize;
 	if (blocksize != 0)
-		blk_shift = int_log2(blocksize);
+		blk_shift = ulong_log2(blocksize);
 	else
-		blk_shift = int_log2(blksize);
+		blk_shift = ulong_log2(blksize);
 
-	width = int_log10(numblocks);
+	width = ulong_log10(numblocks);
 	if (width > logical_width)
 		logical_width = width;
 	if (verbose)
@@ -517,7 +517,7 @@ out_close:
 
 static void usage(const char *progname)
 {
-	fprintf(stderr, "Usage: %s [-b{blocksize}] [-BeksvxX] file ...\n",
+	fprintf(stderr, "Usage: %s [-b{blocksize}[KMG]] [-BeksvxX] file ...\n",
 		progname);
 	exit(1);
 }
@@ -535,7 +535,9 @@ int main(int argc, char**argv)
 		case 'b':
 			if (optarg) {
 				char *end;
-				blocksize = strtoul(optarg, &end, 0);
+				unsigned long val;
+
+				val = strtoul(optarg, &end, 0);
 				if (end) {
 #if __GNUC_PREREQ (7, 0)
 #pragma GCC diagnostic push
@@ -544,15 +546,15 @@ int main(int argc, char**argv)
 					switch (end[0]) {
 					case 'g':
 					case 'G':
-						blocksize *= 1024;
+						val *= 1024;
 						/* fall through */
 					case 'm':
 					case 'M':
-						blocksize *= 1024;
+						val *= 1024;
 						/* fall through */
 					case 'k':
 					case 'K':
-						blocksize *= 1024;
+						val *= 1024;
 						break;
 					default:
 						break;
@@ -561,6 +563,16 @@ int main(int argc, char**argv)
 #pragma GCC diagnostic pop
 #endif
 				}
+				/* Specifying too large a blocksize will just
+				 * shift all extents down to zero length. Even
+				 * 1GB is questionable, but caveat emptor. */
+				if (val > 1024 * 1024 * 1024) {
+					fprintf(stderr,
+						"%s: blocksize %lu over 1GB\n",
+						argv[0], val);
+					usage(argv[0]);
+				}
+				blocksize = val;
 			} else { /* Allow -b without argument for compat. Remove
 				  * this eventually so "-b {blocksize}" works */
 				fprintf(stderr, "%s: -b needs a blocksize "
-- 
1.8.0

[PATCH] misc: handle very large files with filefrag

[Andreas Dilger]

Avoid overflowing the column-width calc printing files over 4B blocks.

Document the [KMG] suffixes for the "-b <blocksize>" option.

The blocksize is limited to at most 1GiB blocksize to avoid shifting
all extents down to zero GB in size.  Even the use of 1GB blocksize
is unlikely, but non-ext4 filesystems may use multi-GB extents.

Signed-off-by: Andreas Dilger <adilger@dilger.ca>
Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197
---
 misc/filefrag.8.in |  4 ++--
 misc/filefrag.c    | 36 ++++++++++++++++++++++++------------
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/misc/filefrag.8.in b/misc/filefrag.8.in
index 5292672..4b89e72 100644
--- a/misc/filefrag.8.in
+++ b/misc/filefrag.8.in
@@ -33,8 +33,8 @@ testing purposes.
 .BI \-b blocksize
 Use
 .I blocksize
-in bytes for output instead of the filesystem blocksize.
-For compatibility with earlier versions of
+in bytes, or with [KMG] suffix, up to 1GB for output instead of the
+filesystem blocksize.  For compatibility with earlier versions of
 .BR filefrag ,
 if
 .I blocksize
diff --git a/misc/filefrag.c b/misc/filefrag.c
index 1eec146..032535f 100644
--- a/misc/filefrag.c
+++ b/misc/filefrag.c
@@ -53,7 +53,7 @@ extern int optind;
 #include <ext2fs/fiemap.h>
 
 int verbose = 0;
-int blocksize;		/* Use specified blocksize (default 1kB) */
+unsigned int blocksize;	/* Use specified blocksize (default 1kB) */
 int sync_file = 0;	/* fsync file before getting the mapping */
 int xattr_map = 0;	/* get xattr mapping */
 int force_bmap;	/* force use of FIBMAP instead of FIEMAP */
@@ -73,7 +73,7 @@ const char *hex_fmt = "%4d: %*llx..%*llx: %*llx..%*llx: %6llx: %s\n";
 #define	EXT4_EXTENTS_FL			0x00080000 /* Inode uses extents */
 #define	EXT3_IOC_GETFLAGS		_IOR('f', 1, long)
 
-static int int_log2(int arg)
+static int ulong_log2(unsigned long arg)
 {
 	int     l = 0;
 
@@ -85,7 +85,7 @@ static int int_log2(int arg)
 	return l;
 }
 
-static int int_log10(unsigned long long arg)
+static int ulong_log10(unsigned long long arg)
 {
 	int     l = 0;
 
@@ -452,17 +452,17 @@ static int frag_report(const char *filename)
 	}
 	last_device = st.st_dev;
 
-	width = int_log10(fsinfo.f_blocks);
+	width = ulong_log10(fsinfo.f_blocks);
 	if (width > physical_width)
 		physical_width = width;
 
 	numblocks = (st.st_size + blksize - 1) / blksize;
 	if (blocksize != 0)
-		blk_shift = int_log2(blocksize);
+		blk_shift = ulong_log2(blocksize);
 	else
-		blk_shift = int_log2(blksize);
+		blk_shift = ulong_log2(blksize);
 
-	width = int_log10(numblocks);
+	width = ulong_log10(numblocks);
 	if (width > logical_width)
 		logical_width = width;
 	if (verbose)
@@ -517,7 +517,7 @@ out_close:
 
 static void usage(const char *progname)
 {
-	fprintf(stderr, "Usage: %s [-b{blocksize}] [-BeksvxX] file ...\n",
+	fprintf(stderr, "Usage: %s [-b{blocksize}[KMG]] [-BeksvxX] file ...\n",
 		progname);
 	exit(1);
 }
@@ -535,7 +535,9 @@ int main(int argc, char**argv)
 		case 'b':
 			if (optarg) {
 				char *end;
-				blocksize = strtoul(optarg, &end, 0);
+				unsigned long val;
+
+				val = strtoul(optarg, &end, 0);
 				if (end) {
 #if __GNUC_PREREQ (7, 0)
 #pragma GCC diagnostic push
@@ -544,15 +546,15 @@ int main(int argc, char**argv)
 					switch (end[0]) {
 					case 'g':
 					case 'G':
-						blocksize *= 1024;
+						val *= 1024;
 						/* fall through */
 					case 'm':
 					case 'M':
-						blocksize *= 1024;
+						val *= 1024;
 						/* fall through */
 					case 'k':
 					case 'K':
-						blocksize *= 1024;
+						val *= 1024;
 						break;
 					default:
 						break;
@@ -561,6 +563,16 @@ int main(int argc, char**argv)
 #pragma GCC diagnostic pop
 #endif
 				}
+				/* Specifying too large a blocksize will just
+				 * shift all extents down to zero length. Even
+				 * 1GB is questionable, but caveat emptor. */
+				if (val > 1024 * 1024 * 1024) {
+					fprintf(stderr,
+						"%s: blocksize %lu over 1GB\n",
+						argv[0], val);
+					usage(argv[0]);
+				}
+				blocksize = val;
 			} else { /* Allow -b without argument for compat. Remove
 				  * this eventually so "-b {blocksize}" works */
 				fprintf(stderr, "%s: -b needs a blocksize "
-- 
1.8.0

[PATCH] e2fsck: avoid overflow with very large dirs

[Andreas Dilger]

In alloc_size_dir() it multiples signed ints when allocating the
buffer for rehashing an htree-indexed directory.  This will overflow
when the directory size is above 4GB, which is possible with largedir
directories having about 100M entries, assuming an average 3/4 leaf
fullness and 24-byte filenames, or fewer with longer filenames.
The same problem exisgs in get_next_block().

Similarly, the out_dir struct used a signed int for the number of
blocks in the directory, which may result in a negative size if the
directory is over 2GB (about 50M entries or fewer).

Use appropriate unsigned variables for block counts, and use larger
types for calculating the byte count for memory offsets/sizes.

Such large directories not been seen yet, but are not too far away.
The ext2fs_get_array() function will properly calculate the needed
memory allocation, and detect overflow on 32-bit systems.
Add ext2fs_resize_array() to do the same for array resize.

Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197
---
 e2fsck/pass2.c      | 10 ++++----
 e2fsck/rehash.c     | 72 +++++++++++++++++++++++++++++------------------------
 lib/ext2fs/ext2fs.h | 44 ++++++++++++++++++++++++++------
 3 files changed, 81 insertions(+), 45 deletions(-)

diff --git a/e2fsck/pass2.c b/e2fsck/pass2.c
index 0fa6233..a280619 100644
--- a/e2fsck/pass2.c
+++ b/e2fsck/pass2.c
@@ -88,7 +88,7 @@ struct check_dir_struct {
 static void update_parents(struct dx_dir_info *dx_dir, int type)
 {
 	struct dx_dirblock_info *dx_db, *dx_parent, *dx_previous;
-	int b;
+	blk_t b;
 
 	for (b = 0, dx_db = dx_dir->dx_block;
 	     b < dx_dir->numblocks;
@@ -130,7 +130,7 @@ void e2fsck_pass2(e2fsck_t ctx)
 	struct check_dir_struct cd;
 	struct dx_dir_info	*dx_dir;
 	struct dx_dirblock_info	*dx_db;
-	int			b;
+	blk_t			b;
 	ext2_ino_t		i;
 	short			depth;
 	problem_t		code;
@@ -570,8 +570,8 @@ static void parse_int_node(ext2_filsys fs,
 			   struct dx_dir_info	*dx_dir,
 			   char *block_buf, int failed_csum)
 {
-	struct 		ext2_dx_root_info  *root;
-	struct 		ext2_dx_entry *ent;
+	struct		ext2_dx_root_info  *root;
+	struct		ext2_dx_entry *ent;
 	struct		ext2_dx_countlimit *limit;
 	struct dx_dirblock_info	*dx_db;
 	int		i, expect_limit, count;
@@ -646,7 +646,7 @@ static void parse_int_node(ext2_filsys fs,
 #endif
 		blk = ext2fs_le32_to_cpu(ent[i].block) & EXT4_DX_BLOCK_MASK;
 		/* Check to make sure the block is valid */
-		if (blk >= (blk_t) dx_dir->numblocks) {
+		if (blk >= dx_dir->numblocks) {
 			cd->pctx.blk = blk;
 			if (fix_problem(cd->ctx, PR_2_HTREE_BADBLK,
 					&cd->pctx))
diff --git a/e2fsck/rehash.c b/e2fsck/rehash.c
index c9d667b..6cb1c8e 100644
--- a/e2fsck/rehash.c
+++ b/e2fsck/rehash.c
@@ -80,8 +80,8 @@ struct fill_dir_struct {
 	errcode_t err;
 	e2fsck_t ctx;
 	struct hash_entry *harray;
-	int max_array, num_array;
-	unsigned int dir_size;
+	blk_t max_array, num_array;
+	ext2_off64_t dir_size;
 	int compress;
 	ext2_ino_t parent;
 	ext2_ino_t dir;
@@ -95,8 +95,8 @@ struct hash_entry {
 };
 
 struct out_dir {
-	int		num;
-	int		max;
+	blk_t		num;
+	blk_t		max;
 	char		*buf;
 	ext2_dirhash_t	*hashes;
 };
@@ -169,13 +169,16 @@ static int fill_dir_block(ext2_filsys fs,
 			continue;
 		}
 		if (fd->num_array >= fd->max_array) {
-			new_array = realloc(fd->harray,
-			    sizeof(struct hash_entry) * (fd->max_array+500));
-			if (!new_array) {
-				fd->err = ENOMEM;
+			errcode_t retval;
+
+			retval = ext2fs_resize_array(sizeof(struct hash_entry),
+						     fd->max_array,
+						     fd->max_array + 500,
+						     &fd->harray);
+			if (retval) {
+				fd->err = retval;
 				return BLOCK_ABORT;
 			}
-			fd->harray = new_array;
 			fd->max_array += 500;
 		}
 		ent = fd->harray + fd->num_array++;
@@ -256,23 +259,28 @@ static EXT2_QSORT_TYPE hash_cmp(const void *a, const void *b)
 }
 
 static errcode_t alloc_size_dir(ext2_filsys fs, struct out_dir *outdir,
-				int blocks)
+				blk_t blocks)
 {
-	void			*new_mem;
+	errcode_t retval;
 
 	if (outdir->max) {
-		new_mem = realloc(outdir->buf, blocks * fs->blocksize);
-		if (!new_mem)
-			return ENOMEM;
-		outdir->buf = new_mem;
-		new_mem = realloc(outdir->hashes,
-				  blocks * sizeof(ext2_dirhash_t));
-		if (!new_mem)
-			return ENOMEM;
-		outdir->hashes = new_mem;
+		retval = ext2fs_resize_array(fs->blocksize, outdir->max, blocks,
+					     &outdir->buf);
+		if (retval)
+			return retval;
+		retval = ext2fs_resize_array(sizeof(ext2_dirhash_t),
+					     outdir->max, blocks,
+					     &outdir->hashes);
+		if (retval)
+			return retval;
 	} else {
-		outdir->buf = malloc(blocks * fs->blocksize);
-		outdir->hashes = malloc(blocks * sizeof(ext2_dirhash_t));
+		retval = ext2fs_get_array(fs->blocksize, blocks, &outdir->buf);
+		if (retval)
+			return retval;
+		retval = ext2fs_get_array(sizeof(ext2_dirhash_t), blocks,
+					  &outdir->hashes);
+		if (retval)
+			return retval;
 		outdir->num = 0;
 	}
 	outdir->max = blocks;
@@ -297,7 +305,7 @@ static errcode_t get_next_block(ext2_filsys fs, struct out_dir *outdir,
 		if (retval)
 			return retval;
 	}
-	*ret = outdir->buf + (outdir->num++ * fs->blocksize);
+	*ret = outdir->buf + (size_t)outdir->num++ * fs->blocksize;
 	memset(*ret, 0, fs->blocksize);
 	return 0;
 }
@@ -367,8 +375,8 @@ static int duplicate_search_and_fix(e2fsck_t ctx, ext2_filsys fs,
 				    struct fill_dir_struct *fd)
 {
 	struct problem_context	pctx;
-	struct hash_entry 	*ent, *prev;
-	int			i, j;
+	struct hash_entry	*ent, *prev;
+	blk_t			i, j;
 	int			fixed = 0;
 	char			new_name[256];
 	unsigned int		new_len;
@@ -869,14 +877,14 @@ errcode_t e2fsck_rehash_dir(e2fsck_t ctx, ext2_ino_t ino,
 	   (inode.i_flags & EXT4_INLINE_DATA_FL))
 		return 0;
 
-	retval = ENOMEM;
-	dir_buf = malloc(inode.i_size);
-	if (!dir_buf)
+	retval = ext2fs_get_mem(inode.i_size, &dir_buf);
+	if (retval)
 		goto errout;
 
 	fd.max_array = inode.i_size / 32;
-	fd.harray = malloc(fd.max_array * sizeof(struct hash_entry));
-	if (!fd.harray)
+	retval = ext2fs_get_array(sizeof(struct hash_entry),
+				  fd.max_array, &fd.harray);
+	if (retval)
 		goto errout;
 
 	fd.ino = ino;
@@ -965,8 +973,8 @@ resort:
 	else
 		retval = e2fsck_check_rebuild_extents(ctx, ino, &inode, pctx);
 errout:
-	free(dir_buf);
-	free(fd.harray);
+	ext2fs_free_mem(&dir_buf);
+	ext2fs_free_mem(&fd.harray);
 
 	free_out_dir(&outdir);
 	return retval;
diff --git a/lib/ext2fs/ext2fs.h b/lib/ext2fs/ext2fs.h
index 59fd974..452af6a 100644
--- a/lib/ext2fs/ext2fs.h
+++ b/lib/ext2fs/ext2fs.h
@@ -1760,6 +1760,8 @@ extern errcode_t ext2fs_get_arrayzero(unsigned long count,
 extern errcode_t ext2fs_free_mem(void *ptr);
 extern errcode_t ext2fs_resize_mem(unsigned long old_size,
 				   unsigned long size, void *ptr);
+extern errcode_t ext2fs_resize_array(unsigned long old_count, unsigned long count,
+				     unsigned long size, void *ptr);
 extern void ext2fs_mark_super_dirty(ext2_filsys fs);
 extern void ext2fs_mark_changed(ext2_filsys fs);
 extern int ext2fs_test_changed(ext2_filsys fs);
@@ -1837,7 +1839,8 @@ _INLINE_ errcode_t ext2fs_get_memzero(unsigned long size, void *ptr)
 	return 0;
 }
 
-_INLINE_ errcode_t ext2fs_get_array(unsigned long count, unsigned long size, void *ptr)
+_INLINE_ errcode_t ext2fs_get_array(unsigned long count, unsigned long size,
+				    void *ptr)
 {
 	if (count && (~0UL)/count < size)
 		return EXT2_ET_NO_MEMORY;
@@ -1847,15 +1850,10 @@ _INLINE_ errcode_t ext2fs_get_array(unsigned long count, unsigned long size, voi
 _INLINE_ errcode_t ext2fs_get_arrayzero(unsigned long count,
 					unsigned long size, void *ptr)
 {
-	void *pp;
-
 	if (count && (~0UL)/count < size)
 		return EXT2_ET_NO_MEMORY;
-	pp = calloc(count, size);
-	if (!pp)
-		return EXT2_ET_NO_MEMORY;
-	memcpy(ptr, &pp, sizeof(pp));
-	return 0;
+
+	return ext2fs_get_memzero((size_t)count * size, ptr);
 }
 
 /*
@@ -1889,6 +1887,36 @@ _INLINE_ errcode_t ext2fs_resize_mem(unsigned long EXT2FS_ATTR((unused)) old_siz
 	memcpy(ptr, &p, sizeof(p));
 	return 0;
 }
+
+/*
+ *  Resize array.  The 'ptr' arg must point to a pointer.
+ */
+_INLINE_ errcode_t ext2fs_resize_array(unsigned long size,
+				       unsigned long old_count,
+				       unsigned long count, void *ptr)
+{
+	unsigned long old_size;
+	errcode_t retval;
+
+	if (count && (~0UL)/count < size)
+		return EXT2_ET_NO_MEMORY;
+
+	size *= count;
+	old_size = size * old_count;
+	retval = ext2fs_resize_mem(old_size, size, ptr);
+	if (retval)
+		return retval;
+
+	if (size > old_size) {
+		void *p;
+
+		memcpy(&p, ptr, sizeof(p));
+		memset((char *)p + old_size, 0, size - old_size);
+		memcpy(ptr, &p, sizeof(p));
+	}
+
+	return 0;
+}
 #endif	/* Custom memory routines */
 
 /*
-- 
1.8.0

Re: [PATCH] misc: handle very large files with filefrag

[Andreas Dilger]

[-- Attachment #1: Type: text/plain, Size: 755 bytes --]

On Feb 11, 2020, at 5:58 PM, Andreas Dilger <adilger@whamcloud.com> wrote:
> 
> Avoid overflowing the column-width calc printing files over 4B blocks.
> 
> Document the [KMG] suffixes for the "-b <blocksize>" option.
> 
> The blocksize is limited to at most 1GiB blocksize to avoid shifting
> all extents down to zero GB in size.  Even the use of 1GB blocksize
> is unlikely, but non-ext4 filesystems may use multi-GB extents.
> 
> Signed-off-by: Andreas Dilger <adilger@dilger.ca>
> Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197

Sorry,
this one was a repeat.  I actually meant to actually send the other patch
"e2fsck: avoid overflow with very large dirs", which was accidentally
dropped from my previous patch series.

Cheers, Andreas






[-- Attachment #2: Message signed with OpenPGP --]
[-- Type: application/pgp-signature, Size: 873 bytes --]

Re: [PATCH 1/9] e2fsck: fix e2fsck_allocate_memory() overflow

[Theodore Y. Ts'o]

On Thu, Feb 06, 2020 at 06:09:38PM -0700, Andreas Dilger wrote:
> e2fsck_allocate_memory() takes an "unsigned int size" argument, which
> will overflow for allocations above 4GB.  This happens for dir_info
> and dx_dir_info arrays when there are more than 350M directories in a
> filesystem, and for the dblist array above 180M directories.
> 
> There is also a risk of overflow during the binary search in both
> e2fsck_get_dir_info() and e2fsck_get_dx_dir_info() when the midpoint
> of the array is calculated, if there would be more than 2B directories
> in the filesystem and working above the half way point.
> 
> Also, in some places inode numbers are "int" instead of "ext2_ino_t",
> which can also cause problems with the array size calculations, and
> makes it hard to identify where inode numbers are used.
> 
> Fix e2fsck_allocate_memory() to take an "unsigned long" argument to
> match ext2fs_get_mem(), so that it can do single memory allocations
> over 4GB.
> 
> Fix e2fsck_get_dir_info() and e2fsck_get_dx_dir_info() to temporarily
> use an unsigned long long value to calculate the midpoint (which will
> always fit into an ext2_ino_t again afterward).
> 
> Change variables that hold inode numbers to be ext2_ino_t, and print
> them as unsigned values instead of printing negative inode numbers.
> 
> Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
> Reviewed-by: Shilong Wang <wshilong@ddn.com>
> Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197

Applied, thanks.

					- Ted

Re: [PATCH 2/9] e2fsck: use proper types for variables

[Theodore Y. Ts'o]

On Thu, Feb 06, 2020 at 06:09:39PM -0700, Andreas Dilger wrote:
> Use ext2_ino_t instead of ino_t for referencing inode numbers.
> Use loff_t for for file offsets, and dgrp_t for group numbers.
> 
> Cast products to ssize_t before multiplication to avoid overflow.
> 
> Signed-off-by: Andreas Dilger <adilger@dilger.ca>
> Reviewed-by: Shilong Wang <wshilong@ddn.com>
> Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197

Applied, thanks.

					- Ted

Re: [PATCH 3/9] e2fsck: avoid mallinfo() if over 2GB allocated

[Theodore Y. Ts'o]

On Thu, Feb 06, 2020 at 06:09:40PM -0700, Andreas Dilger wrote:
> Don't use mallinfo() for determining the amount of memory used if it
> is over 2GB.  Otherwise, the signed ints used by this interface can
> can overflow and return garbage values.  This makes the actual amount
> of memory used by e2fsck misleading and hard to determine.
> 
> Instead, use brk() to get the total amount of memory allocated, and print
> this if the more detailed mallinfo() information is not suitable for use.
> 
> There does not appear to be a mallinfo64() variant of this function.
> There does appear to be an abomination named malloc_info() that writes
> XML-formatted malloc stats to a FILE stream that would need to be read
> and parsed in order to get these stats, but that doesn't seem worthwhile.
> 
> Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
> Reviewed-by: Shilong Wang <wshilong@ddn.com>
> Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197

Applied, thanks.

					- Ted

Re: [PATCH 4/9] e2fsck: reduce memory usage for many directories

[Theodore Y. Ts'o]

On Thu, Feb 06, 2020 at 06:09:41PM -0700, Andreas Dilger wrote:
> Pack struct dx_dir_info and dx_dirblock_info properly in memory, to
> avoid holes, and fields are not larger than necessary.  This reduces
> the memory needed for each hashed dir, according to pahole(1) from:
> 
>     struct dx_dir_info {
>         /* size: 32, cachelines: 1, members: 6 */
>         /* sum members: 26, holes: 1, sum holes: 2 */
>         /* padding: 4 */
>     };
>     struct dx_dirblock_info {
>         /* size: 56, cachelines: 1, members: 9 */
>         /* sum members: 48, holes: 2, sum holes: 8 */
>         /* last cacheline: 56 bytes */
>     };
> 
> to 8 bytes less for each directory and directory block, and leaves
> space for future use if needed (e.g. larger numblocks):
> 
>     struct dx_dir_info {
>         /* size: 24, cachelines: 1, members: 6 */
>         /* sum members: 20, holes: 1, sum holes: 4 */
>         /* bit holes: 1, sum bit holes: 7 bits */
>     };
>     struct dx_dirblock_info {
>         /* size: 48, cachelines: 1, members: 9 */
>     };
> 
> Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
> Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197

Applied, thanks.

					- Ted

Re: [PATCH 5/9] debugfs: allow comment lines in command file

[Theodore Y. Ts'o]

On Thu, Feb 06, 2020 at 06:09:42PM -0700, Andreas Dilger wrote:
> Allow comment lines with '#' at the start of the line in the command
> file passed in to debugfs via the "-f" option or from standard input.
> 
> Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
> Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197

Thanks, applied.

					- Ted

Re: [PATCH 6/9] debugfs: print inode numbers as unsigned

[Theodore Y. Ts'o]

On Thu, Feb 06, 2020 at 06:09:43PM -0700, Andreas Dilger wrote:
> Print inode numbers as unsigned values, to avoid printing negative
> numbers for inodes above 2B.
> 
> Flags should be printed as hex instead of signed decimal values.
> 
> Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
> Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197

Applied, thanks.

						- Ted

Re: [PATCH 7/9] e2fsck: fix overflow if more than 4B inodes

[Theodore Y. Ts'o]

On Thu, Feb 06, 2020 at 06:09:44PM -0700, Andreas Dilger wrote:
> Even though we don't have support for filesystems with over 4B inodes
> in the current e2fsprogs, this may happen in the future.  There are
> latent overflow bugs when calculating the number of inodes in the
> filesystem that can trivially be fixed now, rather than waiting for
> them to be hit at some point in the future.  The block number calcs
> are already correct in this code.
> 
> Signed-off-by: Andreas Dilger <adilger@dilger.ca>
> Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197

Applied, thanks.

						- Ted

Re: [PATCH 8/9] e2fsck: consistently use ext2fs_get_mem()

[Theodore Y. Ts'o]

On Thu, Feb 06, 2020 at 06:09:45PM -0700, Andreas Dilger wrote:
> Consistently use ext2fs_get_mem() and ext2fs_free_mem() instead of
> calling malloc() and free() directly in e2fsck.  In several places
> it is possible to use ext2fs_get_memzero() instead of explicitly
> calling memset() on the memory afterward.
> 
> This is just a code cleanup, and does not fix any specific bugs.
> 
> Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
> Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197

Applied, thanks.

					- Ted

Re: [PATCH 8/9] e2fsck: consistently use ext2fs_get_mem()

[Theodore Y. Ts'o]

On Sat, Feb 29, 2020 at 06:36:02PM -0500, Theodore Y. Ts'o wrote:
> On Thu, Feb 06, 2020 at 06:09:45PM -0700, Andreas Dilger wrote:
> > Consistently use ext2fs_get_mem() and ext2fs_free_mem() instead of
> > calling malloc() and free() directly in e2fsck.  In several places
> > it is possible to use ext2fs_get_memzero() instead of explicitly
> > calling memset() on the memory afterward.
> > 
> > This is just a code cleanup, and does not fix any specific bugs.
> > 
> > Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
> > Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197
> 
> Applied, thanks.

I had to fix up this Makefile so that "make check" would properly link
tst_region.

						- Ted

Re: [PATCH 9/9] misc: handle very large files with filefrag

[Theodore Y. Ts'o]

On Thu, Feb 06, 2020 at 06:09:46PM -0700, Andreas Dilger wrote:
> Avoid overflowing the column-width calc printing files over 4B blocks.
> 
> Document the [KMG] suffixes for the "-b <blocksize>" option.
> 
> The blocksize is limited to at most 1GiB blocksize to avoid shifting
> all extents down to zero GB in size.  Even the use of 1GB blocksize
> is unlikely, but non-ext4 filesystems may use multi-GB extents.
> 
> Signed-off-by: Andreas Dilger <adilger@dilger.ca>
> Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197

Applied, thanks.

				- Ted

Re: [PATCH] e2fsck: avoid overflow with very large dirs

[Theodore Y. Ts'o]

On Tue, Feb 11, 2020 at 06:07:21PM -0700, Andreas Dilger wrote:
> In alloc_size_dir() it multiples signed ints when allocating the
> buffer for rehashing an htree-indexed directory.  This will overflow
> when the directory size is above 4GB, which is possible with largedir
> directories having about 100M entries, assuming an average 3/4 leaf
> fullness and 24-byte filenames, or fewer with longer filenames.
> The same problem exisgs in get_next_block().
> 
> Similarly, the out_dir struct used a signed int for the number of
> blocks in the directory, which may result in a negative size if the
> directory is over 2GB (about 50M entries or fewer).
> 
> Use appropriate unsigned variables for block counts, and use larger
> types for calculating the byte count for memory offsets/sizes.
> 
> Such large directories not been seen yet, but are not too far away.
> The ext2fs_get_array() function will properly calculate the needed
> memory allocation, and detect overflow on 32-bit systems.
> Add ext2fs_resize_array() to do the same for array resize.
> 
> Signed-off-by: Andreas Dilger <adilger@whamcloud.com>
> Lustre-bug-id: https://jira.whamcloud.com/browse/LU-13197

Applied, thanks.  I had to make a slight change to fix a "merge
conflict" with the patch.

					- Ted