[RFC 2/2] squashfs parallel decompression, z_stream per cpu

["J. R. Okajima" <hooanon05@yahoo.co.jp>]

Convert z_stream in squashfs_sb_info into the module-global per-cpu var,
and remove read_data_mutex in squashfs_sb_info.
Also convert the repeated sequence of
	zlib_inflateInit - zlib_inflate - zlib_inflateEnd
into
	zlib_inflateReset - zlib_inflate

zlib_inflateEnd() is not called since its current implementation is less
important.

Signed-off-by: J. R. Okajima <hooanon05@yahoo.co.jp>
---
 fs/squashfs/block.c          |   64 +++++++++++++++++------------------------
 fs/squashfs/cache.c          |    1 +
 fs/squashfs/dir.c            |    1 +
 fs/squashfs/export.c         |    1 +
 fs/squashfs/file.c           |    1 +
 fs/squashfs/fragment.c       |    1 +
 fs/squashfs/id.c             |    1 +
 fs/squashfs/inode.c          |    1 +
 fs/squashfs/namei.c          |    1 +
 fs/squashfs/squashfs.h       |    3 ++
 fs/squashfs/squashfs_fs_sb.h |    2 -
 fs/squashfs/super.c          |   48 ++++++++++++++++++++++++-------
 fs/squashfs/symlink.c        |    1 +
 13 files changed, 76 insertions(+), 50 deletions(-)

diff --git a/fs/squashfs/block.c b/fs/squashfs/block.c
index 1017b94..ed34979 100644
--- a/fs/squashfs/block.c
+++ b/fs/squashfs/block.c
@@ -33,6 +33,7 @@
 #include <linux/string.h>
 #include <linux/buffer_head.h>
 #include <linux/zlib.h>
+#include <linux/percpu.h>
 
 #include "squashfs_fs.h"
 #include "squashfs_fs_sb.h"
@@ -159,20 +160,28 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index,
 	k = 0;
 
 	if (compressed) {
-		int zlib_err = 0, zlib_init = 0;
+		int zlib_err = 0;
+		z_stream *z;
 
 		/*
 		 * Uncompress block.
 		 */
+		/* it disables preemption */
+		z = &get_cpu_var(squashfs_zstream);
+		zlib_err = zlib_inflateReset(z);
+		if (zlib_err != Z_OK) {
+			put_cpu_var(squashfs_zstream);
+			ERROR("zlib_inflateReset returned"
+			      " unexpected result 0x%x, srclength %d\n",
+			      zlib_err, srclength);
+			goto block_release;
+		}
 
-		mutex_lock(&msblk->read_data_mutex);
-
-		msblk->stream.avail_out = 0;
-		msblk->stream.avail_in = 0;
-
+		z->avail_out = 0;
+		z->avail_in = 0;
 		bytes = length;
 		do {
-			if (msblk->stream.avail_in == 0 && k < b) {
+			if (z->avail_in == 0 && k < b) {
 				avail = min(bytes, msblk->devblksize - offset);
 				bytes -= avail;
 
@@ -182,46 +191,30 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index,
 					continue;
 				}
 
-				msblk->stream.next_in = bh[k]->b_data + offset;
-				msblk->stream.avail_in = avail;
+				z->next_in = bh[k]->b_data + offset;
+				z->avail_in = avail;
 				offset = 0;
 			}
 
-			if (msblk->stream.avail_out == 0 && page < pages) {
-				msblk->stream.next_out = buffer[page++];
-				msblk->stream.avail_out = PAGE_CACHE_SIZE;
+			if (z->avail_out == 0 && page < pages) {
+				z->next_out = buffer[page++];
+				z->avail_out = PAGE_CACHE_SIZE;
 			}
 
-			if (!zlib_init) {
-				zlib_err = zlib_inflateInit(&msblk->stream);
-				if (zlib_err != Z_OK) {
-					ERROR("zlib_inflateInit returned"
-						" unexpected result 0x%x,"
-						" srclength %d\n", zlib_err,
-						srclength);
-					goto release_mutex;
-				}
-				zlib_init = 1;
-			}
+			zlib_err = zlib_inflate(z, Z_SYNC_FLUSH);
 
-			zlib_err = zlib_inflate(&msblk->stream, Z_SYNC_FLUSH);
-
-			if (msblk->stream.avail_in == 0 && k < b)
+			if (z->avail_in == 0 && k < b)
 				put_bh(bh[k++]);
 		} while (zlib_err == Z_OK);
 
 		if (zlib_err != Z_STREAM_END) {
+			put_cpu_var(squashfs_zstream);
 			ERROR("zlib_inflate error, data probably corrupt\n");
-			goto release_mutex;
+			goto block_release;
 		}
 
-		zlib_err = zlib_inflateEnd(&msblk->stream);
-		if (zlib_err != Z_OK) {
-			ERROR("zlib_inflate error, data probably corrupt\n");
-			goto release_mutex;
-		}
-		length = msblk->stream.total_out;
-		mutex_unlock(&msblk->read_data_mutex);
+		length = z->total_out;
+		put_cpu_var(squashfs_zstream);
 	} else {
 		/*
 		 * Block is uncompressed.
@@ -252,9 +245,6 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index,
 	kfree(bh);
 	return length;
 
-release_mutex:
-	mutex_unlock(&msblk->read_data_mutex);
-
 block_release:
 	for (; k < b; k++)
 		put_bh(bh[k]);
diff --git a/fs/squashfs/cache.c b/fs/squashfs/cache.c
index 40c98fa..d90814f 100644
--- a/fs/squashfs/cache.c
+++ b/fs/squashfs/cache.c
@@ -53,6 +53,7 @@
 #include <linux/wait.h>
 #include <linux/zlib.h>
 #include <linux/pagemap.h>
+#include <linux/percpu.h>
 
 #include "squashfs_fs.h"
 #include "squashfs_fs_sb.h"
diff --git a/fs/squashfs/dir.c b/fs/squashfs/dir.c
index 566b0ea..3d2a632 100644
--- a/fs/squashfs/dir.c
+++ b/fs/squashfs/dir.c
@@ -31,6 +31,7 @@
 #include <linux/vfs.h>
 #include <linux/slab.h>
 #include <linux/zlib.h>
+#include <linux/percpu.h>
 
 #include "squashfs_fs.h"
 #include "squashfs_fs_sb.h"
diff --git a/fs/squashfs/export.c b/fs/squashfs/export.c
index 2b1b8fe..9b33473 100644
--- a/fs/squashfs/export.c
+++ b/fs/squashfs/export.c
@@ -41,6 +41,7 @@
 #include <linux/exportfs.h>
 #include <linux/zlib.h>
 #include <linux/slab.h>
+#include <linux/percpu.h>
 
 #include "squashfs_fs.h"
 #include "squashfs_fs_sb.h"
diff --git a/fs/squashfs/file.c b/fs/squashfs/file.c
index 717767d..f9728f3 100644
--- a/fs/squashfs/file.c
+++ b/fs/squashfs/file.c
@@ -48,6 +48,7 @@
 #include <linux/pagemap.h>
 #include <linux/mutex.h>
 #include <linux/zlib.h>
+#include <linux/percpu.h>
 
 #include "squashfs_fs.h"
 #include "squashfs_fs_sb.h"
diff --git a/fs/squashfs/fragment.c b/fs/squashfs/fragment.c
index b5a2c15..fd6f776 100644
--- a/fs/squashfs/fragment.c
+++ b/fs/squashfs/fragment.c
@@ -37,6 +37,7 @@
 #include <linux/vfs.h>
 #include <linux/slab.h>
 #include <linux/zlib.h>
+#include <linux/percpu.h>
 
 #include "squashfs_fs.h"
 #include "squashfs_fs_sb.h"
diff --git a/fs/squashfs/id.c b/fs/squashfs/id.c
index 3795b83..cea22c1 100644
--- a/fs/squashfs/id.c
+++ b/fs/squashfs/id.c
@@ -35,6 +35,7 @@
 #include <linux/vfs.h>
 #include <linux/slab.h>
 #include <linux/zlib.h>
+#include <linux/percpu.h>
 
 #include "squashfs_fs.h"
 #include "squashfs_fs_sb.h"
diff --git a/fs/squashfs/inode.c b/fs/squashfs/inode.c
index 9101dbd..6977d34 100644
--- a/fs/squashfs/inode.c
+++ b/fs/squashfs/inode.c
@@ -41,6 +41,7 @@
 #include <linux/fs.h>
 #include <linux/vfs.h>
 #include <linux/zlib.h>
+#include <linux/percpu.h>
 
 #include "squashfs_fs.h"
 #include "squashfs_fs_sb.h"
diff --git a/fs/squashfs/namei.c b/fs/squashfs/namei.c
index 9e39865..3a6e51f 100644
--- a/fs/squashfs/namei.c
+++ b/fs/squashfs/namei.c
@@ -58,6 +58,7 @@
 #include <linux/string.h>
 #include <linux/dcache.h>
 #include <linux/zlib.h>
+#include <linux/percpu.h>
 
 #include "squashfs_fs.h"
 #include "squashfs_fs_sb.h"
diff --git a/fs/squashfs/squashfs.h b/fs/squashfs/squashfs.h
index 0e9feb6..0ff530e 100644
--- a/fs/squashfs/squashfs.h
+++ b/fs/squashfs/squashfs.h
@@ -70,6 +70,9 @@ extern struct inode *squashfs_iget(struct super_block *, long long,
 				unsigned int);
 extern int squashfs_read_inode(struct inode *, long long);
 
+/* super.c */
+DECLARE_PER_CPU(z_stream, squashfs_zstream);
+
 /*
  * Inodes and files operations
  */
diff --git a/fs/squashfs/squashfs_fs_sb.h b/fs/squashfs/squashfs_fs_sb.h
index c8c6561..95489cc 100644
--- a/fs/squashfs/squashfs_fs_sb.h
+++ b/fs/squashfs/squashfs_fs_sb.h
@@ -61,10 +61,8 @@ struct squashfs_sb_info {
 	__le64			*id_table;
 	__le64			*fragment_index;
 	unsigned int		*fragment_index_2;
-	struct mutex		read_data_mutex;
 	struct mutex		meta_index_mutex;
 	struct meta_index	*meta_index;
-	z_stream		stream;
 	__le64			*inode_lookup_table;
 	u64			inode_table;
 	u64			directory_table;
diff --git a/fs/squashfs/super.c b/fs/squashfs/super.c
index 6c197ef..e78ffbf 100644
--- a/fs/squashfs/super.c
+++ b/fs/squashfs/super.c
@@ -37,6 +37,7 @@
 #include <linux/module.h>
 #include <linux/zlib.h>
 #include <linux/magic.h>
+#include <linux/percpu.h>
 
 #include "squashfs_fs.h"
 #include "squashfs_fs_sb.h"
@@ -87,13 +88,6 @@ static int squashfs_fill_super(struct super_block *sb, void *data, int silent)
 	}
 	msblk = sb->s_fs_info;
 
-	msblk->stream.workspace = kmalloc(zlib_inflate_workspacesize(),
-		GFP_KERNEL);
-	if (msblk->stream.workspace == NULL) {
-		ERROR("Failed to allocate zlib workspace\n");
-		goto failure;
-	}
-
 	sblk = kzalloc(sizeof(*sblk), GFP_KERNEL);
 	if (sblk == NULL) {
 		ERROR("Failed to allocate squashfs_super_block\n");
@@ -103,7 +97,6 @@ static int squashfs_fill_super(struct super_block *sb, void *data, int silent)
 	msblk->devblksize = sb_min_blocksize(sb, BLOCK_SIZE);
 	msblk->devblksize_log2 = ffz(~msblk->devblksize);
 
-	mutex_init(&msblk->read_data_mutex);
 	mutex_init(&msblk->meta_index_mutex);
 
 	/*
@@ -295,14 +288,12 @@ failed_mount:
 	kfree(msblk->inode_lookup_table);
 	kfree(msblk->fragment_index);
 	kfree(msblk->id_table);
-	kfree(msblk->stream.workspace);
 	kfree(sb->s_fs_info);
 	sb->s_fs_info = NULL;
 	kfree(sblk);
 	return err;
 
 failure:
-	kfree(msblk->stream.workspace);
 	kfree(sb->s_fs_info);
 	sb->s_fs_info = NULL;
 	return -ENOMEM;
@@ -349,7 +340,6 @@ static void squashfs_put_super(struct super_block *sb)
 		kfree(sbi->id_table);
 		kfree(sbi->fragment_index);
 		kfree(sbi->meta_index);
-		kfree(sbi->stream.workspace);
 		kfree(sb->s_fs_info);
 		sb->s_fs_info = NULL;
 	}
@@ -394,14 +384,43 @@ static void destroy_inodecache(void)
 }
 
 
+DEFINE_PER_CPU(z_stream, squashfs_zstream);
 static int __init init_squashfs_fs(void)
 {
 	int err = init_inodecache();
+	int cpu, sz;
+	z_stream *z;
 
 	if (err)
 		return err;
 
+	err = -ENOMEM;
+	z = NULL; /* suppress gcc warning */
+	sz = zlib_inflate_workspacesize();
+	for_each_online_cpu(cpu) {
+		z = &per_cpu(squashfs_zstream, cpu);
+		z->workspace = kmalloc(sz, GFP_KERNEL);
+		if (!z->workspace) {
+			ERROR("Failed to allocate zlib workspace\n");
+			break;
+		}
+		err = zlib_inflateInit(z);
+		if (err == Z_MEM_ERROR) {
+			err = -ENOMEM;
+			ERROR("Failed to intialize zlib\n");
+			break;
+		}
+	}
+	if (!z->workspace) {
+		for_each_online_cpu(cpu) {
+			z = &per_cpu(squashfs_zstream, cpu);
+			kfree(z->workspace);
+		}
+		goto out_err;
+	}
+
 	err = register_filesystem(&squashfs_fs_type);
+out_err:
 	if (err) {
 		destroy_inodecache();
 		return err;
@@ -416,7 +435,14 @@ static int __init init_squashfs_fs(void)
 
 static void __exit exit_squashfs_fs(void)
 {
+	int cpu;
+	z_stream *z;
+
 	unregister_filesystem(&squashfs_fs_type);
+	for_each_online_cpu(cpu) {
+		z = &per_cpu(squashfs_zstream, cpu);
+		kfree(z->workspace);
+	}
 	destroy_inodecache();
 }
 
diff --git a/fs/squashfs/symlink.c b/fs/squashfs/symlink.c
index 83d8788..133776b 100644
--- a/fs/squashfs/symlink.c
+++ b/fs/squashfs/symlink.c
@@ -37,6 +37,7 @@
 #include <linux/string.h>
 #include <linux/pagemap.h>
 #include <linux/zlib.h>
+#include <linux/percpu.h>
 
 #include "squashfs_fs.h"
 #include "squashfs_fs_sb.h"
-- 
1.6.6.1