All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH 00/13] erofs_utils: new mkfs framework
@ 2019-05-31  0:50 Gao Xiang
  2019-05-31  0:50 ` [PATCH 01/13] erofs-utils: add erofs on-disk layout Gao Xiang
                   ` (12 more replies)
  0 siblings, 13 replies; 14+ messages in thread
From: Gao Xiang @ 2019-05-31  0:50 UTC (permalink / raw)


Hi all,
I spent nearly a month to make big changes to erofs-mkfs,
which mainly introduces a new mkfs framework.

I personally think it is much cleaner and more scalable than
the old one, which is hack and not clean enough.

In-place compression is still work-in-progress, thus it isn't
included in this series.

Comments are welcome!

Thanks,
Gao Xiang

Gao Xiang (8):
  erofs-utils: introduce buffer cache
  erofs-utils: introduce inode operations
  erofs-utils: introduce generic compression framework
  erofs-utils: introduce lz4/lz4hc compression algorithm
  erofs-utils: introduce compression for regular files
  erofs-utils: propagate compressed size to its callers
  erofs-utils: add a README
  erofs-utils: fix potential erofs_bh_balloon failure

Li Guifu (5):
  erofs-utils: add erofs on-disk layout
  erofs-utils: introduce erofs-utils basic headers
  erofs-utils: introduce miscellaneous files
  erofs-utils: add input/output functions
  erofs-utils: introduce mkfs support

-- 
2.17.1

^ permalink raw reply	[flat|nested] 14+ messages in thread
* [PATCH 01/13] erofs-utils: add erofs on-disk layout
  2019-05-31  0:50 [PATCH 00/13] erofs_utils: new mkfs framework Gao Xiang
@ 2019-05-31  0:50 ` Gao Xiang
  2019-05-31  0:50 ` [PATCH 02/13] erofs-utils: introduce erofs-utils basic headers Gao Xiang
                   ` (11 subsequent siblings)
  12 siblings, 0 replies; 14+ messages in thread
From: Gao Xiang @ 2019-05-31  0:50 UTC (permalink / raw)


From: Li Guifu <bluce.liguifu@huawei.com>

This patch adds a header file describing the erofs on-disk layout, which
should be kept in line with the kernel implementation all the time.

Signed-off-by: Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Fang Wei <fangwei1 at huawei.com>
Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 include/erofs_fs.h | 275 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 275 insertions(+)
 create mode 100644 include/erofs_fs.h

diff --git a/include/erofs_fs.h b/include/erofs_fs.h
new file mode 100644
index 0000000..4b6c1d1
--- /dev/null
+++ b/include/erofs_fs.h
@@ -0,0 +1,275 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Apache-2.0 */
+/*
+ * erofs_utils/include/erofs_fs.h
+ *
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ *
+ * This file is dual-licensed; you may select either the GNU General Public
+ * License version 2 or Apache License, Version 2.0. See the file COPYING
+ * in the main directory of the Linux distribution for more details.
+ */
+#ifndef __EROFS_FS_H
+#define __EROFS_FS_H
+
+/* Enhanced(Extended) ROM File System */
+#define EROFS_SUPER_MAGIC_V1    0xE0F5E1E2
+#define EROFS_SUPER_OFFSET      1024
+
+struct erofs_super_block {
+/*  0 */__le32 magic;           /* in the little endian */
+/*  4 */__le32 checksum;        /* crc32c(super_block) */
+/*  8 */__le32 features;
+/* 12 */__u8 blkszbits;         /* support block_size == PAGE_SIZE only */
+/* 13 */__u8 reserved;
+
+/* 14 */__le16 root_nid;
+/* 16 */__le64 inos;            /* total valid ino # (== f_files - f_favail) */
+
+/* 24 */__le64 build_time;      /* inode v1 time derivation */
+/* 32 */__le32 build_time_nsec;
+/* 36 */__le32 blocks;          /* used for statfs */
+/* 40 */__le32 meta_blkaddr;
+/* 44 */__le32 xattr_blkaddr;
+/* 48 */__u8 uuid[16];          /* 128-bit uuid for volume */
+/* 64 */__u8 volume_name[16];   /* volume name */
+
+/* 80 */__u8 reserved2[48];     /* 128 bytes */
+} __packed;
+
+/*
+ * erofs inode data mapping:
+ * 0 - inode plain without inline data A:
+ * inode, [xattrs], ... | ... | no-holed data
+ * 1 - inode VLE compression B:
+ * inode, [xattrs], extents ... | ...
+ * 2 - inode plain with inline data C:
+ * inode, [xattrs], last_inline_data, ... | ... | no-holed data
+ * 3~7 - reserved
+ */
+enum {
+	EROFS_INODE_LAYOUT_PLAIN,
+	EROFS_INODE_LAYOUT_COMPRESSION,
+	EROFS_INODE_LAYOUT_INLINE,
+	EROFS_INODE_LAYOUT_MAX
+};
+
+/* bit definitions of inode i_advise */
+#define EROFS_I_VERSION_BITS            1
+#define EROFS_I_DATA_MAPPING_BITS       3
+
+#define EROFS_I_VERSION_BIT             0
+#define EROFS_I_DATA_MAPPING_BIT        1
+
+struct erofs_inode_v1 {
+/*  0 */__le16 i_advise;
+
+/* 1 header + n-1 * 4 bytes inline xattr to keep continuity */
+/*  2 */__le16 i_xattr_icount;
+/*  4 */__le16 i_mode;
+/*  6 */__le16 i_nlink;
+/*  8 */__le32 i_size;
+/* 12 */__le32 i_reserved;
+/* 16 */union {
+		/* file total compressed blocks for data mapping 1 */
+		__le32 compressed_blocks;
+		__le32 raw_blkaddr;
+
+		/* for device files, used to indicate old/new device # */
+		__le32 rdev;
+	} i_u __packed;
+/* 20 */__le32 i_ino;           /* only used for 32-bit stat compatibility */
+/* 24 */__le16 i_uid;
+/* 26 */__le16 i_gid;
+/* 28 */__le32 i_checksum;
+} __packed;
+
+/* 32 bytes on-disk inode */
+#define EROFS_INODE_LAYOUT_V1   0
+/* 64 bytes on-disk inode */
+#define EROFS_INODE_LAYOUT_V2   1
+
+struct erofs_inode_v2 {
+	__le16 i_advise;
+
+	/* 1 header + n-1 * 4 bytes inline xattr to keep continuity */
+	__le16 i_xattr_icount;
+	__le16 i_mode;
+	__le16 i_reserved;      /* 8 bytes */
+	__le64 i_size;          /* 16 bytes */
+	union {
+		/* file total compressed blocks for data mapping 1 */
+		__le32 compressed_blocks;
+		__le32 raw_blkaddr;
+
+		/* for device files, used to indicate old/new device # */
+		__le32 rdev;
+	} i_u __packed;
+
+	/* only used for 32-bit stat compatibility */
+	__le32 i_ino;           /* 24 bytes */
+
+	__le32 i_uid;
+	__le32 i_gid;
+	__le64 i_ctime;         /* 32 bytes */
+	__le32 i_ctime_nsec;
+	__le32 i_nlink;
+	__u8   i_reserved2[12];
+	__le32 i_checksum;      /* 64 bytes */
+} __packed;
+
+#define EROFS_MAX_SHARED_XATTRS         (128)
+/* h_shared_count between 129 ... 255 are special # */
+#define EROFS_SHARED_XATTR_EXTENT       (255)
+
+/*
+ * inline xattrs (n == i_xattr_icount):
+ * erofs_xattr_ibody_header(1) + (n - 1) * 4 bytes
+ *          12 bytes           /                   \
+ *                            /                     \
+ *                           /-----------------------\
+ *                           |  erofs_xattr_entries+ |
+ *                           +-----------------------+
+ * inline xattrs must starts in erofs_xattr_ibody_header,
+ * for read-only fs, no need to introduce h_refcount
+ */
+struct erofs_xattr_ibody_header {
+	__le32 h_checksum;
+	__u8   h_shared_count;
+	__u8   h_reserved[7];
+	__le32 h_shared_xattrs[0];      /* shared xattr id array */
+} __packed;
+
+/* Name indexes */
+#define EROFS_XATTR_INDEX_USER              1
+#define EROFS_XATTR_INDEX_POSIX_ACL_ACCESS  2
+#define EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT 3
+#define EROFS_XATTR_INDEX_TRUSTED           4
+#define EROFS_XATTR_INDEX_LUSTRE            5
+#define EROFS_XATTR_INDEX_SECURITY          6
+
+/* xattr entry (for both inline & shared xattrs) */
+struct erofs_xattr_entry {
+	__u8   e_name_len;      /* length of name */
+	__u8   e_name_index;    /* attribute name index */
+	__le16 e_value_size;    /* size of attribute value */
+	/* followed by e_name and e_value */
+	char   e_name[0];       /* attribute name */
+} __packed;
+
+#define ondisk_xattr_ibody_size(count)	({\
+	u32 __count = le16_to_cpu(count); \
+	((__count) == 0) ? 0 : \
+	sizeof(struct erofs_xattr_ibody_header) + \
+		sizeof(__u32) * ((__count) - 1); })
+
+#define EROFS_XATTR_ALIGN(size) round_up(size, sizeof(struct erofs_xattr_entry))
+#define EROFS_XATTR_ENTRY_SIZE(entry) EROFS_XATTR_ALIGN( \
+	sizeof(struct erofs_xattr_entry) + \
+	(entry)->e_name_len + le16_to_cpu((entry)->e_value_size))
+
+/* have to be aligned with 8 bytes on disk */
+struct erofs_extent_header {
+	__le32 eh_checksum;
+	__le32 eh_reserved[3];
+} __packed;
+
+/*
+ * Z_EROFS Variable-sized Logical Extent cluster type:
+ *    0 - literal (uncompressed) cluster
+ *    1 - compressed cluster (for the head logical cluster)
+ *    2 - compressed cluster (for the other logical clusters)
+ *
+ * In detail,
+ *    0 - literal (uncompressed) cluster,
+ *        di_advise = 0
+ *        di_clusterofs = the literal data offset of the cluster
+ *        di_blkaddr = the blkaddr of the literal cluster
+ *
+ *    1 - compressed cluster (for the head logical cluster)
+ *        di_advise = 1
+ *        di_clusterofs = the decompressed data offset of the cluster
+ *        di_blkaddr = the blkaddr of the compressed cluster
+ *
+ *    2 - compressed cluster (for the other logical clusters)
+ *        di_advise = 2
+ *        di_clusterofs =
+ *           the decompressed data offset in its own head cluster
+ *        di_u.delta[0] = distance to its corresponding head cluster
+ *        di_u.delta[1] = distance to its corresponding tail cluster
+ *                (di_advise could be 0, 1 or 2)
+ */
+enum {
+	Z_EROFS_VLE_CLUSTER_TYPE_PLAIN,
+	Z_EROFS_VLE_CLUSTER_TYPE_HEAD,
+	Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD,
+	Z_EROFS_VLE_CLUSTER_TYPE_RESERVED,
+	Z_EROFS_VLE_CLUSTER_TYPE_MAX
+};
+
+#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS        2
+#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT         0
+
+struct z_erofs_vle_decompressed_index {
+	__le16 di_advise;
+	/* where to decompress in the head cluster */
+	__le16 di_clusterofs;
+
+	union {
+		/* for the head cluster */
+		__le32 blkaddr;
+		/*
+		 * for the rest clusters
+		 * eg. for 4k page-sized cluster, maximum 4K*64k = 256M)
+		 * [0] - pointing to the head cluster
+		 * [1] - pointing to the tail cluster
+		 */
+		__le16 delta[2];
+	} di_u __packed;		/* 8 bytes */
+} __packed;
+
+#define Z_EROFS_VLE_EXTENT_ALIGN(size) round_up(size, \
+	sizeof(struct z_erofs_vle_decompressed_index))
+
+/* dirent sorts in alphabet order, thus we can do binary search */
+struct erofs_dirent {
+	__le64 nid;     /*  0, node number */
+	__le16 nameoff; /*  8, start offset of file name */
+	__u8 file_type; /* 10, file type */
+	__u8 reserved;  /* 11, reserved */
+} __packed;
+
+/* file types used in inode_info->flags */
+enum {
+	EROFS_FT_UNKNOWN,
+	EROFS_FT_REG_FILE,
+	EROFS_FT_DIR,
+	EROFS_FT_CHRDEV,
+	EROFS_FT_BLKDEV,
+	EROFS_FT_FIFO,
+	EROFS_FT_SOCK,
+	EROFS_FT_SYMLINK,
+	EROFS_FT_MAX
+};
+
+#define EROFS_NAME_LEN      255
+
+/* check the EROFS on-disk layout strictly at compile time */
+static inline void erofs_check_ondisk_layout_definitions(void)
+{
+	BUILD_BUG_ON(sizeof(struct erofs_super_block) != 128);
+	BUILD_BUG_ON(sizeof(struct erofs_inode_v1) != 32);
+	BUILD_BUG_ON(sizeof(struct erofs_inode_v2) != 64);
+	BUILD_BUG_ON(sizeof(struct erofs_xattr_ibody_header) != 12);
+	BUILD_BUG_ON(sizeof(struct erofs_xattr_entry) != 4);
+	BUILD_BUG_ON(sizeof(struct erofs_extent_header) != 16);
+	BUILD_BUG_ON(sizeof(struct z_erofs_vle_decompressed_index) != 8);
+	BUILD_BUG_ON(sizeof(struct erofs_dirent) != 12);
+
+	BUILD_BUG_ON(BIT(Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) <
+		     Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1);
+}
+
+#endif
+
-- 
2.17.1

^ permalink raw reply related	[flat|nested] 14+ messages in thread
* [PATCH 02/13] erofs-utils: introduce erofs-utils basic headers
  2019-05-31  0:50 [PATCH 00/13] erofs_utils: new mkfs framework Gao Xiang
  2019-05-31  0:50 ` [PATCH 01/13] erofs-utils: add erofs on-disk layout Gao Xiang
@ 2019-05-31  0:50 ` Gao Xiang
  2019-05-31  0:50 ` [PATCH 03/13] erofs-utils: introduce miscellaneous files Gao Xiang
                   ` (10 subsequent siblings)
  12 siblings, 0 replies; 14+ messages in thread
From: Gao Xiang @ 2019-05-31  0:50 UTC (permalink / raw)


From: Li Guifu <bluce.liguifu@huawei.com>

This patch adds basic definitions, and a simple
kernel-like list implementaion.

Signed-off-by: Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Fang Wei <fangwei1 at huawei.com>
Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 include/erofs/defs.h     | 159 +++++++++++++++++++++++++++++++++++++++
 include/erofs/err.h      |  34 +++++++++
 include/erofs/internal.h | 126 +++++++++++++++++++++++++++++++
 include/erofs/list.h     | 110 +++++++++++++++++++++++++++
 4 files changed, 429 insertions(+)
 create mode 100644 include/erofs/defs.h
 create mode 100644 include/erofs/err.h
 create mode 100644 include/erofs/internal.h
 create mode 100644 include/erofs/list.h

diff --git a/include/erofs/defs.h b/include/erofs/defs.h
new file mode 100644
index 0000000..111b703
--- /dev/null
+++ b/include/erofs/defs.h
@@ -0,0 +1,159 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/defs.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ * Modified by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#ifndef __EROFS_DEFS_H
+#define __EROFS_DEFS_H
+
+#include <stddef.h>
+#include <stdint.h>
+#include <assert.h>
+#include <inttypes.h>
+
+#include <stdbool.h>
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifdef HAVE_LINUX_TYPES_H
+#include <linux/types.h>
+#endif
+
+/*
+ * container_of - cast a member of a structure out to the containing structure
+ * @ptr:	the pointer to the member.
+ * @type:	the type of the container struct this is embedded in.
+ * @member:	the name of the member within the struct.
+ */
+#define container_of(ptr, type, member) ({			\
+	const typeof(((type *)0)->member) *__mptr = (ptr);	\
+	(type *)((char *)__mptr - offsetof(type, member)); })
+
+typedef uint8_t         u8;
+typedef uint16_t        u16;
+typedef uint32_t        u32;
+typedef uint64_t        u64;
+
+#ifndef HAVE_LINUX_TYPES_H
+typedef u8	__u8;
+typedef u16	__u16;
+typedef u32	__u32;
+typedef u64	__u64;
+typedef u16	__le16;
+typedef u32	__le32;
+typedef u64	__le64;
+typedef u16	__be16;
+typedef u32	__be32;
+typedef u64	__be64;
+#endif
+
+typedef int8_t          s8;
+typedef int16_t         s16;
+typedef int32_t         s32;
+typedef int64_t         s64;
+
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+/*
+ * The host byte order is the same as network byte order,
+ * so these functions are all just identity.
+ */
+#define cpu_to_le16(x) ((__u16)(x))
+#define cpu_to_le32(x) ((__u32)(x))
+#define cpu_to_le64(x) ((__u64)(x))
+#define le16_to_cpu(x) ((__u16)(x))
+#define le32_to_cpu(x) ((__u32)(x))
+#define le64_to_cpu(x) ((__u64)(x))
+
+#else
+#if __BYTE_ORDER == __BIG_ENDIAN
+#define cpu_to_le16(x) (__builtin_bswap16(x))
+#define cpu_to_le32(x) (__builtin_bswap32(x))
+#define cpu_to_le64(x) (__builtin_bswap64(x))
+#define le16_to_cpu(x) (__builtin_bswap16(x))
+#define le32_to_cpu(x) (__builtin_bswap32(x))
+#define le64_to_cpu(x) (__builtin_bswap64(x))
+#else
+#pragma error
+#endif
+#endif
+
+#ifndef __OPTIMIZE__
+#define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2 * !!(condition)]))
+#else
+#define BUILD_BUG_ON(condition) assert(condition)
+#endif
+
+#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
+
+#define __round_mask(x, y)      ((__typeof__(x))((y)-1))
+#define round_up(x, y)          ((((x)-1) | __round_mask(x, y))+1)
+#define round_down(x, y)        ((x) & ~__round_mask(x, y))
+
+/* The `const' in roundup() prevents gcc-3.3 from calling __divdi3 */
+#define roundup(x, y) (					\
+{							\
+	const typeof(y) __y = y;			\
+	(((x) + (__y - 1)) / __y) * __y;		\
+}							\
+)
+#define rounddown(x, y) (				\
+{							\
+	typeof(x) __x = (x);				\
+	__x - (__x % (y));				\
+}							\
+)
+
+#define min(x, y) ({				\
+	typeof(x) _min1 = (x);			\
+	typeof(y) _min2 = (y);			\
+	(void) (&_min1 == &_min2);		\
+	_min1 < _min2 ? _min1 : _min2; })
+
+#define max(x, y) ({				\
+	typeof(x) _max1 = (x);			\
+	typeof(y) _max2 = (y);			\
+	(void) (&_max1 == &_max2);		\
+	_max1 > _max2 ? _max1 : _max2; })
+
+/*
+ * ..and if you can't take the strict types, you can specify one yourself.
+ * Or don't use min/max at all, of course.
+ */
+#define min_t(type, x, y) ({			\
+	type __min1 = (x);			\
+	type __min2 = (y);			\
+	__min1 < __min2 ? __min1: __min2; })
+
+#define max_t(type, x, y) ({			\
+	type __max1 = (x);			\
+	type __max2 = (y);			\
+	__max1 > __max2 ? __max1: __max2; })
+
+#define ARRAY_SIZE(arr)	(sizeof(arr) / sizeof((arr)[0]))
+
+#define BIT(nr)             (1UL << (nr))
+#define BIT_ULL(nr)         (1ULL << (nr))
+#define BIT_MASK(nr)        (1UL << ((nr) % BITS_PER_LONG))
+#define BIT_WORD(nr)        ((nr) / BITS_PER_LONG)
+#define BIT_ULL_MASK(nr)    (1ULL << ((nr) % BITS_PER_LONG_LONG))
+#define BIT_ULL_WORD(nr)    ((nr) / BITS_PER_LONG_LONG)
+#define BITS_PER_BYTE       8
+#define BITS_TO_LONGS(nr)   DIV_ROUND_UP(nr, BITS_PER_BYTE * sizeof(long))
+
+#define BUG_ON(cond)        assert(!(cond))
+
+#ifdef NDEBUG
+#define DBG_BUGON(condition)	((void)(condition))
+#else
+#define DBG_BUGON(condition)	BUG_ON(condition)
+#endif
+
+#endif
+
diff --git a/include/erofs/err.h b/include/erofs/err.h
new file mode 100644
index 0000000..fd4c873
--- /dev/null
+++ b/include/erofs/err.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/err.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_ERR_H
+#define __EROFS_ERR_H
+
+#include <errno.h>
+
+#define MAX_ERRNO (4095)
+#define IS_ERR_VALUE(x)                                                        \
+	((unsigned long)(void *)(x) >= (unsigned long)-MAX_ERRNO)
+
+static inline void *ERR_PTR(long error)
+{
+	return (void *)error;
+}
+
+static inline int IS_ERR(const void *ptr)
+{
+	return IS_ERR_VALUE((unsigned long)ptr);
+}
+
+static inline long PTR_ERR(const void *ptr)
+{
+	return (long) ptr;
+}
+
+#endif
+
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
new file mode 100644
index 0000000..238a076
--- /dev/null
+++ b/include/erofs/internal.h
@@ -0,0 +1,126 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/internal.h
+ *
+ * Copyright (C) 2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#ifndef __EROFS_INTERNAL_H
+#define __EROFS_INTERNAL_H
+
+#include "list.h"
+#include "err.h"
+
+typedef unsigned short umode_t;
+typedef unsigned int __u32;
+
+#define __packed __attribute__((__packed__))
+
+#include "erofs_fs.h"
+#include <fcntl.h>
+
+#ifndef PATH_MAX
+#define PATH_MAX        4096    /* # chars in a path name including nul */
+#endif
+
+#define PAGE_SHIFT		(12)
+#define PAGE_SIZE		(1U << PAGE_SHIFT)
+
+#define LOG_BLOCK_SIZE          (12)
+#define EROFS_BLKSIZ            (1U << LOG_BLOCK_SIZE)
+
+#define EROFS_ISLOTBITS		5
+#define EROFS_SLOTSIZE		(1U << EROFS_ISLOTBITS)
+
+typedef u64 erofs_off_t;
+typedef u64 erofs_nid_t;
+/* data type for filesystem-wide blocks number */
+typedef u32 erofs_blk_t;
+
+#define NULL_ADDR	((unsigned int)-1)
+#define NULL_ADDR_UL	((unsigned long)-1)
+
+#define erofs_blknr(addr)       ((addr) / EROFS_BLKSIZ)
+#define erofs_blkoff(addr)      ((addr) % EROFS_BLKSIZ)
+#define blknr_to_addr(nr)       ((erofs_off_t)(nr) * EROFS_BLKSIZ)
+
+#define BLK_ROUND_UP(addr)	DIV_ROUND_UP(addr, EROFS_BLKSIZ)
+
+struct erofs_buffer_head;
+
+struct erofs_sb_info {
+	erofs_blk_t meta_blkaddr;
+	erofs_blk_t xattr_blkaddr;
+};
+
+/* global sbi */
+extern struct erofs_sb_info sbi;
+
+struct erofs_inode {
+	struct list_head i_hash, i_subdirs;
+
+	unsigned int i_count;
+	struct erofs_inode *i_parent;
+
+	umode_t i_mode;
+	erofs_off_t i_size;
+
+	u64 i_ino[2];
+	u32 i_uid;
+	u32 i_gid;
+	u64 i_ctime;
+	u32 i_ctime_nsec;
+	u32 i_nlink;
+
+	union {
+		u32 i_blkaddr;
+		u32 i_blocks;
+		u32 i_rdev;
+	} u;
+
+	char i_srcpath[PATH_MAX + 1];
+
+	unsigned char data_mapping_mode;
+	bool compression_disabled;
+
+	unsigned char inode_isize;
+	/* inline tail-end packing size */
+	unsigned short idata_size;
+
+	unsigned int xattr_isize;
+	unsigned int extent_isize;
+
+	erofs_nid_t nid;
+	struct erofs_buffer_head *bh;
+	struct erofs_buffer_head *bh_inline, *bh_data;
+
+	void *idata;
+};
+
+#define IS_ROOT(x)	((x) == (x)->i_parent)
+
+struct erofs_dentry {
+	struct list_head d_child;	/* child of parent list */
+
+	unsigned int type;
+	char name[EROFS_NAME_LEN];
+	union {
+		struct erofs_inode *inode;
+		erofs_nid_t nid;
+	};
+};
+
+#include <stdio.h>
+#include <string.h>
+
+static inline const char *erofs_strerror(int err)
+{
+	static char msg[256];
+
+	sprintf(msg, "[Error %d] %s", -err, strerror(-err));
+	return msg;
+}
+
+#endif
+
diff --git a/include/erofs/list.h b/include/erofs/list.h
new file mode 100644
index 0000000..e290843
--- /dev/null
+++ b/include/erofs/list.h
@@ -0,0 +1,110 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/list.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_LIST_HEAD_H
+#define __EROFS_LIST_HEAD_H
+
+#include "defs.h"
+
+struct list_head {
+	struct list_head *prev;
+	struct list_head *next;
+};
+
+#define LIST_HEAD_INIT(name)                                                   \
+	{                                                                      \
+		&(name), &(name)                                               \
+	}
+
+#define LIST_HEAD(name) struct list_head name = LIST_HEAD_INIT(name)
+
+static inline void init_list_head(struct list_head *list)
+{
+	list->prev = list;
+	list->next = list;
+}
+
+static inline void __list_add(struct list_head *entry,
+			      struct list_head *prev,
+			      struct list_head *next)
+{
+	entry->prev = prev;
+	entry->next = next;
+	prev->next  = entry;
+	next->prev  = entry;
+}
+
+static inline void list_add(struct list_head *entry, struct list_head *head)
+{
+	__list_add(entry, head, head->next);
+}
+
+static inline void list_add_tail(struct list_head *entry,
+				 struct list_head *head)
+{
+	__list_add(entry, head->prev, head);
+}
+
+static inline void __list_del(struct list_head *prev, struct list_head *next)
+{
+	prev->next = next;
+	next->prev = prev;
+}
+
+static inline void list_del(struct list_head *entry)
+{
+	__list_del(entry->prev, entry->next);
+	entry->prev = entry->next = NULL;
+}
+
+static inline int list_empty(struct list_head *head)
+{
+	return head->next == head;
+}
+
+#define list_entry(ptr, type, member) container_of(ptr, type, member)
+
+#define list_first_entry(ptr, type, member)                                    \
+	list_entry((ptr)->next, type, member)
+
+#define list_last_entry(ptr, type, member)                                     \
+	list_entry((ptr)->prev, type, member)
+
+#define list_next_entry(pos, member)                                           \
+	list_entry((pos)->member.next, typeof(*(pos)), member)
+
+#define list_prev_entry(pos, member)                                           \
+	list_entry((pos)->member.prev, typeof(*(pos)), member)
+
+#define list_for_each(pos, head)                                               \
+	for (pos = (head)->next; pos != (head); pos = pos->next)
+
+#define list_for_each_safe(pos, n, head)                                       \
+	for (pos = (head)->next, n = pos->next; pos != (head);                 \
+	     pos = n, n = pos->next)
+
+#define list_for_each_entry(pos, head, member)                                 \
+	for (pos = list_first_entry(head, typeof(*pos), member);               \
+	     &pos->member != (head);                                           \
+	     pos = list_next_entry(pos, member))
+
+#define list_for_each_entry_reverse(pos, head, member)                         \
+	for (pos = list_last_entry(head, typeof(*pos), member);               \
+	     &pos->member != (head);                                           \
+	     pos = list_prev_entry(pos, member))
+
+#define list_for_each_entry_from(pos, head, member)                            \
+	for (; &pos->member != (head); pos = list_next_entry(pos, member))
+
+#define list_for_each_entry_safe(pos, n, head, member)                         \
+	for (pos = list_first_entry(head, typeof(*pos), member),               \
+	    n    = list_next_entry(pos, member);                               \
+	     &pos->member != (head);                                           \
+	     pos = n, n = list_next_entry(n, member))
+
+#endif
-- 
2.17.1

^ permalink raw reply related	[flat|nested] 14+ messages in thread
* [PATCH 03/13] erofs-utils: introduce miscellaneous files
  2019-05-31  0:50 [PATCH 00/13] erofs_utils: new mkfs framework Gao Xiang
  2019-05-31  0:50 ` [PATCH 01/13] erofs-utils: add erofs on-disk layout Gao Xiang
  2019-05-31  0:50 ` [PATCH 02/13] erofs-utils: introduce erofs-utils basic headers Gao Xiang
@ 2019-05-31  0:50 ` Gao Xiang
  2019-05-31  0:50 ` [PATCH 04/13] erofs-utils: add input/output functions Gao Xiang
                   ` (9 subsequent siblings)
  12 siblings, 0 replies; 14+ messages in thread
From: Gao Xiang @ 2019-05-31  0:50 UTC (permalink / raw)


From: Li Guifu <bluce.liguifu@huawei.com>

This patch introduces a global configuration scheme, some
functions which can print messages in different level,
build scripts and .gitignore file.

Signed-off-by: Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Fang Wei <fangwei1 at huawei.com>
Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 .gitignore             |  25 ++++++++++
 Makefile.am            |   6 +++
 VERSION                |   2 +
 autogen.sh             |   9 ++++
 configure.ac           | 107 +++++++++++++++++++++++++++++++++++++++++
 include/erofs/config.h |  31 ++++++++++++
 include/erofs/print.h  |  67 ++++++++++++++++++++++++++
 lib/Makefile.am        |   7 +++
 lib/config.c           |  36 ++++++++++++++
 9 files changed, 290 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 Makefile.am
 create mode 100644 VERSION
 create mode 100755 autogen.sh
 create mode 100644 configure.ac
 create mode 100644 include/erofs/config.h
 create mode 100644 include/erofs/print.h
 create mode 100644 lib/Makefile.am
 create mode 100644 lib/config.c

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..3a39a1e
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,25 @@
+.*
+*~
+*.diff
+*.o
+*.la
+*.a
+*.patch
+*.rej
+
+#
+# Generated files
+#
+aclocal.m4
+autom4te.cache
+config.*
+Makefile
+Makefile.in
+config/
+m4/
+configure
+configure.scan
+libtool
+stamp-h
+stamp-h1
+
diff --git a/Makefile.am b/Makefile.am
new file mode 100644
index 0000000..ee5fd92
--- /dev/null
+++ b/Makefile.am
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0+
+# Makefile.am
+
+ACLOCAL_AMFLAGS = -I m4
+
+SUBDIRS=lib
diff --git a/VERSION b/VERSION
new file mode 100644
index 0000000..a27f297
--- /dev/null
+++ b/VERSION
@@ -0,0 +1,2 @@
+0.1
+2019-04-20
diff --git a/autogen.sh b/autogen.sh
new file mode 100755
index 0000000..fdda7e1
--- /dev/null
+++ b/autogen.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0+
+
+aclocal && \
+autoheader && \
+autoconf && \
+libtoolize && \
+automake -a -c
+
diff --git a/configure.ac b/configure.ac
new file mode 100644
index 0000000..9c6d8bb
--- /dev/null
+++ b/configure.ac
@@ -0,0 +1,107 @@
+#                                               -*- Autoconf -*-
+# Process this file with autoconf to produce a configure script.
+
+AC_PREREQ([2.69])
+
+m4_define([erofs_utils_version], m4_esyscmd([sed -n '1p' VERSION | tr -d '\n']))
+m4_define([erofs_utils_date], m4_esyscmd([sed -n '2p' VERSION | tr -d '\n']))
+
+AC_INIT([erofs-utils], [erofs_utils_version], [linux-erofs at lists.ozlabs.org])
+AC_CONFIG_SRCDIR([config.h.in])
+AC_CONFIG_HEADERS([config.h])
+AC_CONFIG_MACRO_DIR([m4])
+AC_CONFIG_AUX_DIR(config)
+AM_INIT_AUTOMAKE([foreign -Wall -Werror])
+
+# Checks for programs.
+AM_PROG_AR
+AC_PROG_CC
+AC_PROG_INSTALL
+
+LT_INIT
+
+dnl EROFS_UTILS_PARSE_DIRECTORY
+dnl Input:  $1 = a string to a relative or absolute directory
+dnl Output: $2 = the variable to set with the absolute directory
+AC_DEFUN([EROFS_UTILS_PARSE_DIRECTORY],
+[
+ dnl Check if argument is a directory
+ if test -d $1 ; then
+    dnl Get the absolute path of the directory
+    dnl in case of relative directory.
+    dnl If realpath is not a valid command,
+    dnl an error is produced and we keep the given path.
+    local_tmp=`realpath $1 2>/dev/null`
+    if test "$local_tmp" != "" ; then
+       if test -d "$local_tmp" ; then
+           $2="$local_tmp"
+       else
+           $2=$1
+       fi
+    else
+       $2=$1
+    fi
+    dnl Check for space in the directory
+    if test `echo $1|cut -d' ' -f1` != $1 ; then
+        AC_MSG_ERROR($1 directory shall not contain any space.)
+    fi
+ else
+    AC_MSG_ERROR($1 shall be a valid directory)
+ fi
+])
+
+# Checks for libraries.
+
+# Checks for header files.
+AC_CHECK_HEADERS(m4_flatten([
+	dirent.h
+	fcntl.h
+	inttypes.h
+	linux/types.h
+	limits.h
+	stddef.h
+	stdint.h
+	stdlib.h
+	string.h
+	sys/stat.h
+	sys/time.h
+	unistd.h
+]))
+
+# Checks for typedefs, structures, and compiler characteristics.
+AC_C_INLINE
+AC_TYPE_INT64_T
+AC_TYPE_SIZE_T
+AC_TYPE_SSIZE_T
+AC_CHECK_MEMBERS([struct stat.st_rdev])
+AC_TYPE_UINT64_T
+
+#
+# Check to see if llseek() is declared in unistd.h.  On some libc's
+# it is, and on others it isn't..... Thank you glibc developers....
+#
+AC_CHECK_DECL(llseek,
+  [AC_DEFINE(HAVE_LLSEEK_PROTOTYPE, 1,
+    [Define to 1 if llseek declared in unistd.h])],,
+  [#include <unistd.h>])
+
+#
+# Check to see if lseek64() is declared in unistd.h.  Glibc's header files
+# are so convoluted that I can't tell whether it will always be defined,
+# and if it isn't defined while lseek64 is defined in the library,
+# disaster will strike.
+#
+# Warning!  Use of --enable-gcc-wall may throw off this test.
+#
+AC_CHECK_DECL(lseek64,[AC_DEFINE(HAVE_LSEEK64_PROTOTYPE, 1,
+  [Define to 1 if lseek64 declared in unistd.h])],,
+  [#define _LARGEFILE_SOURCE
+   #define _LARGEFILE64_SOURCE
+   #include <unistd.h>])
+
+# Checks for library functions.
+AC_CHECK_FUNCS([gettimeofday memset realpath strdup strerror strrchr strtoull])
+
+AC_CONFIG_FILES([Makefile
+		 lib/Makefile])
+AC_OUTPUT
diff --git a/include/erofs/config.h b/include/erofs/config.h
new file mode 100644
index 0000000..e31732b
--- /dev/null
+++ b/include/erofs/config.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/config.h
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_CONFIG_H
+#define __EROFS_CONFIG_H
+
+#include "defs.h"
+
+struct erofs_configure {
+	const char *c_version;
+	int c_dbg_lvl;
+	bool c_dry_run;
+
+	/* related arguments for mkfs.erofs */
+	char *c_img_path;
+	char *c_src_path;
+};
+
+extern struct erofs_configure cfg;
+
+void erofs_init_configure(void);
+void erofs_show_config(void);
+void erofs_exit_configure(void);
+
+#endif
+
diff --git a/include/erofs/print.h b/include/erofs/print.h
new file mode 100644
index 0000000..bc0b8d4
--- /dev/null
+++ b/include/erofs/print.h
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/print.h
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_PRINT_H
+#define __EROFS_PRINT_H
+
+#include "config.h"
+#include <stdio.h>
+
+#define FUNC_LINE_FMT "%s() Line[%d] "
+
+#ifndef pr_fmt
+#define pr_fmt(fmt) "EROFS: " FUNC_LINE_FMT fmt "\n"
+#endif
+
+#define erofs_dbg(fmt, ...) do {				\
+	if (cfg.c_dbg_lvl >= 7) {				\
+		fprintf(stdout,					\
+			pr_fmt(fmt),				\
+			__func__,				\
+			__LINE__,				\
+			##__VA_ARGS__);				\
+	}							\
+} while (0)
+
+#define erofs_info(fmt, ...) do {				\
+	if (cfg.c_dbg_lvl >= 3) {				\
+		fprintf(stdout,					\
+			pr_fmt(fmt),				\
+			__func__,				\
+			__LINE__,				\
+			##__VA_ARGS__);				\
+		fflush(stdout);					\
+	}							\
+} while (0)
+
+#define erofs_warn(fmt, ...) do {				\
+	if (cfg.c_dbg_lvl >= 2) {				\
+		fprintf(stdout,					\
+			pr_fmt(fmt),				\
+			__func__,				\
+			__LINE__,				\
+			##__VA_ARGS__);				\
+		fflush(stdout);					\
+	}							\
+} while (0)
+
+#define erofs_err(fmt, ...) do {				\
+	if (cfg.c_dbg_lvl >= 0) {				\
+		fprintf(stderr,					\
+			"Err: " pr_fmt(fmt),			\
+			__func__,				\
+			__LINE__,				\
+			##__VA_ARGS__);				\
+	}							\
+} while (0)
+
+#define erofs_dump(fmt, ...) fprintf(stderr, fmt, ##__VA_ARGS__)
+
+
+#endif
+
diff --git a/lib/Makefile.am b/lib/Makefile.am
new file mode 100644
index 0000000..6f1da26
--- /dev/null
+++ b/lib/Makefile.am
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0+
+# Makefile.am
+
+noinst_LTLIBRARIES = liberofs.la
+liberofs_la_SOURCES = config.c
+liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
+
diff --git a/lib/config.c b/lib/config.c
new file mode 100644
index 0000000..6ff8e4d
--- /dev/null
+++ b/lib/config.c
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs_utils/lib/config.c
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#include <string.h>
+#include "erofs/print.h"
+
+struct erofs_configure cfg;
+
+void erofs_init_configure(void)
+{
+	memset(&cfg, 0, sizeof(cfg));
+
+	cfg.c_dbg_lvl  = 0;
+	cfg.c_version  = PACKAGE_VERSION;
+	cfg.c_dry_run  = false;
+}
+
+void erofs_show_config(void)
+{
+	const struct erofs_configure *c = &cfg;
+
+	erofs_dump("\tc_version:           [%8s]\n", c->c_version);
+	erofs_dump("\tc_dbg_lvl:           [%8d]\n", c->c_dbg_lvl);
+	erofs_dump("\tc_dry_run:           [%8d]\n", c->c_dry_run);
+}
+
+void erofs_exit_configure(void)
+{
+
+}
+
-- 
2.17.1

^ permalink raw reply related	[flat|nested] 14+ messages in thread
* [PATCH 04/13] erofs-utils: add input/output functions
  2019-05-31  0:50 [PATCH 00/13] erofs_utils: new mkfs framework Gao Xiang
                   ` (2 preceding siblings ...)
  2019-05-31  0:50 ` [PATCH 03/13] erofs-utils: introduce miscellaneous files Gao Xiang
@ 2019-05-31  0:50 ` Gao Xiang
  2019-05-31  0:50 ` [PATCH 05/13] erofs-utils: introduce buffer cache Gao Xiang
                   ` (8 subsequent siblings)
  12 siblings, 0 replies; 14+ messages in thread
From: Gao Xiang @ 2019-05-31  0:50 UTC (permalink / raw)


From: Li Guifu <bluce.liguifu@huawei.com>

This patch adds definitions and functions which are
mainly used for reading and writing target image files.

Signed-off-by: Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Fang Wei <fangwei1 at huawei.com>
Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 include/erofs/io.h |  33 ++++++++++++
 lib/Makefile.am    |   2 +-
 lib/io.c           | 123 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 157 insertions(+), 1 deletion(-)
 create mode 100644 include/erofs/io.h
 create mode 100644 lib/io.c

diff --git a/include/erofs/io.h b/include/erofs/io.h
new file mode 100644
index 0000000..9471388
--- /dev/null
+++ b/include/erofs/io.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/io.h
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_IO_H
+#define __EROFS_IO_H
+
+#include <unistd.h>
+#include "internal.h"
+
+#ifndef O_BINARY
+#define O_BINARY	0
+#endif
+
+int dev_open(const char *devname);
+void dev_close(void);
+int dev_write(const void *buf, u64 offset, size_t len);
+int dev_fsync(void);
+u64 dev_length(void);
+
+static inline int blk_write(const void *buf, erofs_blk_t blkaddr,
+			    u32 nblocks)
+{
+	return dev_write(buf, blknr_to_addr(blkaddr),
+			 blknr_to_addr(nblocks));
+}
+
+#endif
+
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 6f1da26..a2c1b24 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -2,6 +2,6 @@
 # Makefile.am
 
 noinst_LTLIBRARIES = liberofs.la
-liberofs_la_SOURCES = config.c
+liberofs_la_SOURCES = config.c io.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 
diff --git a/lib/io.c b/lib/io.c
new file mode 100644
index 0000000..f624535
--- /dev/null
+++ b/lib/io.c
@@ -0,0 +1,123 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs_utils/lib/io.c
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#define _LARGEFILE64_SOURCE
+#include <sys/stat.h>
+#include "erofs/io.h"
+
+#define pr_fmt(fmt) "EROFS IO: " FUNC_LINE_FMT fmt "\n"
+#include "erofs/print.h"
+
+static const char *erofs_devname;
+static int erofs_devfd = -1;
+static u64 erofs_devsz;
+
+void dev_close(void)
+{
+	close(erofs_devfd);
+	erofs_devname = NULL;
+	erofs_devfd   = -1;
+	erofs_devsz   = 0;
+}
+
+int dev_open(const char *dev)
+{
+	struct stat st;
+	int fd, ret;
+
+	fd = open(dev, O_RDWR | O_CREAT | O_BINARY, 0644);
+	if (fd < 0) {
+		erofs_err("failed to open(%s).", dev);
+		return -errno;
+	}
+
+	ret = fstat(fd, &st);
+	if (ret) {
+		erofs_err("failed to fstat(%s).", dev);
+		close(fd);
+		return -errno;
+	}
+
+	switch (st.st_mode & S_IFMT) {
+	case S_IFBLK:
+		erofs_devsz = st.st_size;
+		break;
+	case S_IFREG:
+		ret = ftruncate(fd, 0);
+		if (ret) {
+			erofs_err("failed to ftruncate(%s).", dev);
+			close(fd);
+			return -errno;
+		}
+		/* INT64_MAX is the limit of kernel vfs */
+		erofs_devsz = INT64_MAX;
+		break;
+	default:
+		erofs_err("bad file type (%s, %o).", dev, st.st_mode);
+		close(fd);
+		return -EINVAL;
+	}
+
+	erofs_devname = dev;
+	erofs_devfd = fd;
+	erofs_devsz = round_down(erofs_devsz, EROFS_BLKSIZ);
+
+	erofs_info("successfully to open %s", dev);
+	return 0;
+}
+
+u64 dev_length(void)
+{
+	return erofs_devsz;
+}
+
+int dev_write(const void *buf, u64 offset, size_t len)
+{
+	int ret;
+
+	if (cfg.c_dry_run)
+		return 0;
+
+	if (!buf) {
+		erofs_err("buf is NULL");
+		return -EINVAL;
+	}
+
+	if (offset >= erofs_devsz || len > erofs_devsz ||
+	    offset > erofs_devsz - len) {
+		erofs_err("Write posion[%" PRIu64 ", %zd] is too large beyond the end of device(%" PRIu64 ").",
+			  offset, len, erofs_devsz);
+		return -EINVAL;
+	}
+
+	ret = pwrite64(erofs_devfd, buf, len, (off64_t)offset);
+	if (ret != (int)len) {
+		if (ret < 0) {
+			erofs_err("Failed to write data into device - %s:[%" PRIu64 ", %zd].",
+				  erofs_devname, offset, len);
+			return -errno;
+		}
+
+		erofs_err("Writing data into device - %s:[%" PRIu64 ", %zd] - was truncated.",
+			  erofs_devname, offset, len);
+		return -ERANGE;
+	}
+	return 0;
+}
+
+int dev_fsync(void)
+{
+	int ret;
+
+	ret = fsync(erofs_devfd);
+	if (ret) {
+		erofs_err("Could not fsync device!!!");
+		return -EIO;
+	}
+	return 0;
+}
-- 
2.17.1

^ permalink raw reply related	[flat|nested] 14+ messages in thread
* [PATCH 05/13] erofs-utils: introduce buffer cache
  2019-05-31  0:50 [PATCH 00/13] erofs_utils: new mkfs framework Gao Xiang
                   ` (3 preceding siblings ...)
  2019-05-31  0:50 ` [PATCH 04/13] erofs-utils: add input/output functions Gao Xiang
@ 2019-05-31  0:50 ` Gao Xiang
  2019-05-31  0:50 ` [PATCH 06/13] erofs-utils: introduce inode operations Gao Xiang
                   ` (7 subsequent siblings)
  12 siblings, 0 replies; 14+ messages in thread
From: Gao Xiang @ 2019-05-31  0:50 UTC (permalink / raw)


From: Gao Xiang <gaoxiang25@huawei.com>

This patch adds a buffer cache mainly to manage
incomplete metadata blocks. mkfs logic is also
simplified with the help of buffer cache.

Signed-off-by: Miao Xie <miaoxie at huawei.com>
[ Gao Xiang: with heavy changes. ]
Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 include/erofs/cache.h | 104 +++++++++++++
 lib/Makefile.am       |   2 +-
 lib/cache.c           | 343 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 448 insertions(+), 1 deletion(-)
 create mode 100644 include/erofs/cache.h
 create mode 100644 lib/cache.c

diff --git a/include/erofs/cache.h b/include/erofs/cache.h
new file mode 100644
index 0000000..108757a
--- /dev/null
+++ b/include/erofs/cache.h
@@ -0,0 +1,104 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/cache.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Miao Xie <miaoxie at huawei.com>
+ * with heavy changes by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#ifndef __EROFS_CACHE_H
+#define __EROFS_CACHE_H
+
+#include "internal.h"
+
+struct erofs_buffer_head;
+struct erofs_buffer_block;
+
+#define DATA		0
+#define META		1
+/* including inline xattrs, extent */
+#define INODE		2
+/* shared xattrs */
+#define XATTR		3
+
+struct erofs_bhops {
+	bool (*preflush)(struct erofs_buffer_head *bh);
+	bool (*flush)(struct erofs_buffer_head *bh);
+};
+
+struct erofs_buffer_head {
+	struct list_head list;
+	struct erofs_buffer_block *block;
+
+	unsigned int off;
+	struct erofs_bhops *op;
+
+	void *fsprivate;
+};
+
+struct erofs_buffer_block {
+	struct list_head list;
+
+	erofs_blk_t blkaddr;
+	int type;
+
+	struct erofs_buffer_head buffers;
+};
+
+static inline const int get_alignsize(int type, int *type_ret)
+{
+	if (type == DATA)
+		return EROFS_BLKSIZ;
+
+	if (type == INODE) {
+		*type_ret = META;
+		return sizeof(struct erofs_inode_v1);
+	} else if (type == XATTR) {
+		*type_ret = META;
+		return sizeof(struct erofs_xattr_entry);
+	}
+
+	if (type == META)
+		return 1;
+	return -EINVAL;
+}
+
+extern struct erofs_bhops erofs_drop_directly_bhops;
+extern struct erofs_bhops erofs_skip_write_bhops;
+extern struct erofs_bhops erofs_buf_write_bhops;
+
+static inline erofs_off_t erofs_btell(struct erofs_buffer_head *bh, bool end)
+{
+	const struct erofs_buffer_block *bb = bh->block;
+
+	if (bb->blkaddr == NULL_ADDR)
+		return NULL_ADDR_UL;
+
+	return blknr_to_addr(bb->blkaddr) +
+		(end ? list_next_entry(bh, list)->off : bh->off);
+}
+
+static inline bool erofs_bh_flush_generic_end(struct erofs_buffer_head *bh)
+{
+	list_del(&bh->list);
+	free(bh);
+	return true;
+}
+
+struct erofs_buffer_head *erofs_buffer_init(void);
+int erofs_bh_balloon(struct erofs_buffer_head *bh, unsigned int incr);
+
+struct erofs_buffer_head *erofs_balloc(int type, unsigned int size,
+				       unsigned int required_ext,
+				       unsigned int inline_ext);
+struct erofs_buffer_head *erofs_battach(struct erofs_buffer_head *bh,
+					int type, int size);
+
+erofs_blk_t erofs_mapbh(struct erofs_buffer_block *bb, bool end);
+bool erofs_bflush(struct erofs_buffer_block *bb);
+
+void erofs_bdrop(struct erofs_buffer_head *bh, bool tryrevoke);
+
+#endif
+
diff --git a/lib/Makefile.am b/lib/Makefile.am
index a2c1b24..8508660 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -2,6 +2,6 @@
 # Makefile.am
 
 noinst_LTLIBRARIES = liberofs.la
-liberofs_la_SOURCES = config.c io.c
+liberofs_la_SOURCES = config.c io.c cache.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 
diff --git a/lib/cache.c b/lib/cache.c
new file mode 100644
index 0000000..967b543
--- /dev/null
+++ b/lib/cache.c
@@ -0,0 +1,343 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs_utils/lib/cache.c
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Miao Xie <miaoxie at huawei.com>
+ * with heavy changes by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#include <stdlib.h>
+#include <erofs/cache.h>
+#include "erofs/io.h"
+#include "erofs/print.h"
+
+static struct erofs_buffer_block blkh = {
+	.list = LIST_HEAD_INIT(blkh.list),
+	.blkaddr = NULL_ADDR,
+};
+static erofs_blk_t tail_blkaddr;
+
+static bool erofs_bh_flush_drop_directly(struct erofs_buffer_head *bh)
+{
+	return erofs_bh_flush_generic_end(bh);
+}
+
+struct erofs_bhops erofs_drop_directly_bhops = {
+	.flush = erofs_bh_flush_drop_directly,
+};
+
+static bool erofs_bh_flush_skip_write(struct erofs_buffer_head *bh)
+{
+	return false;
+}
+
+struct erofs_bhops erofs_skip_write_bhops = {
+	.flush = erofs_bh_flush_skip_write,
+};
+
+int erofs_bh_flush_generic_write(struct erofs_buffer_head *bh, void *buf)
+{
+	struct erofs_buffer_head *nbh = list_next_entry(bh, list);
+	erofs_off_t offset = erofs_btell(bh, false);
+
+	DBG_BUGON(nbh->off < bh->off);
+	return dev_write(buf, offset, nbh->off - bh->off);
+}
+
+static bool erofs_bh_flush_buf_write(struct erofs_buffer_head *bh)
+{
+	int err = erofs_bh_flush_generic_write(bh, bh->fsprivate);
+
+	if (err)
+		return false;
+	free(bh->fsprivate);
+	return erofs_bh_flush_generic_end(bh);
+}
+
+struct erofs_bhops erofs_buf_write_bhops = {
+	.flush = erofs_bh_flush_buf_write,
+};
+
+/* return buffer_head of erofs super block (with size 0) */
+struct erofs_buffer_head *erofs_buffer_init(void)
+{
+	struct erofs_buffer_head *bh = erofs_balloc(META, 0, 0, 0);
+
+	if (IS_ERR(bh))
+		return bh;
+
+	bh->op = &erofs_skip_write_bhops;
+	return bh;
+}
+
+/* return occupied bytes in specific buffer block if succeed */
+static int __erofs_battach(struct erofs_buffer_block *bb,
+			   struct erofs_buffer_head *bh,
+			   unsigned int incr,
+			   unsigned int alignsize,
+			   unsigned int extrasize,
+			   bool dryrun)
+{
+	const unsigned int alignedoffset = roundup(bb->buffers.off, alignsize);
+	const int oob = alignedoffset + incr + extrasize -
+			roundup(bb->buffers.off + 1, EROFS_BLKSIZ);
+	bool tailupdate = false;
+	erofs_blk_t blkaddr;
+
+	if (oob >= 0) {
+		/* the next buffer block should be NULL_ADDR all the time */
+		if (oob && list_next_entry(bb, list)->blkaddr != NULL_ADDR)
+			return -EINVAL;
+
+		blkaddr = bb->blkaddr;
+		if (blkaddr != NULL_ADDR) {
+			tailupdate = (tail_blkaddr == blkaddr +
+				      BLK_ROUND_UP(bb->buffers.off));
+			if (oob && !tailupdate)
+				return -EINVAL;
+		}
+	}
+
+	if (!dryrun) {
+		if (bh) {
+			bh->off = alignedoffset;
+			bh->block = bb;
+			list_add_tail(&bh->list, &bb->buffers.list);
+		}
+		bb->buffers.off = alignedoffset + incr;
+		/* need to update the tail_blkaddr */
+		if (tailupdate)
+			tail_blkaddr = blkaddr + BLK_ROUND_UP(bb->buffers.off);
+	}
+	return (alignedoffset + incr) % EROFS_BLKSIZ;
+}
+
+int erofs_bh_balloon(struct erofs_buffer_head *bh, unsigned int incr)
+{
+	struct erofs_buffer_block *const bb = bh->block;
+
+	/* should be the tail bh in the corresponding buffer block */
+	if (bh->list.next != &bb->buffers.list)
+		return -EINVAL;
+
+	return __erofs_battach(bb, NULL, incr, 1, 0, false);
+}
+
+struct erofs_buffer_head *erofs_balloc(int type, unsigned int size,
+				       unsigned int required_ext,
+				       unsigned int inline_ext)
+{
+	struct erofs_buffer_block *cur, *bb;
+	struct erofs_buffer_head *bh;
+	unsigned int alignsize, used0, usedmax;
+
+	int ret = get_alignsize(type, &type);
+
+	if (ret < 0)
+		return ERR_PTR(ret);
+	alignsize = ret;
+
+	used0 = (size + required_ext) % EROFS_BLKSIZ + inline_ext;
+	usedmax = 0;
+	bb = NULL;
+
+	list_for_each_entry(cur, &blkh.list, list) {
+		unsigned int used_before, used;
+
+		used_before = cur->buffers.off % EROFS_BLKSIZ;
+
+		/* skip if buffer block is just full */
+		if (!used_before)
+			continue;
+
+		/* skip if the entry which has different type */
+		if (cur->type != type)
+			continue;
+
+		ret = __erofs_battach(cur, NULL, size, alignsize,
+				      required_ext + inline_ext, true);
+		if (ret < 0)
+			continue;
+
+		used = (ret + required_ext) % EROFS_BLKSIZ + inline_ext;
+
+		/* should contain inline data in current block */
+		if (used > EROFS_BLKSIZ)
+			continue;
+
+		/*
+		 * remaining should be smaller than before or
+		 * larger than allocating a new buffer block
+		 */
+		if (used < used_before && used < used0)
+			continue;
+
+		if (usedmax < used) {
+			bb = cur;
+			usedmax = used;
+		}
+	}
+
+	if (bb) {
+		bh = malloc(sizeof(struct erofs_buffer_head));
+		if (!bh)
+			return ERR_PTR(-ENOMEM);
+		goto found;
+	}
+
+	/* allocate a new buffer block */
+	if (used0 > EROFS_BLKSIZ)
+		return ERR_PTR(-ENOSPC);
+
+	bb = malloc(sizeof(struct erofs_buffer_block));
+	if (!bb)
+		return ERR_PTR(-ENOMEM);
+
+	bb->type = type;
+	bb->blkaddr = NULL_ADDR;
+	bb->buffers.off = 0;
+	init_list_head(&bb->buffers.list);
+	list_add_tail(&bb->list, &blkh.list);
+
+	bh = malloc(sizeof(struct erofs_buffer_head));
+	if (!bh) {
+		free(bb);
+		return ERR_PTR(-ENOMEM);
+	}
+found:
+	ret = __erofs_battach(bb, bh, size, alignsize,
+			      required_ext + inline_ext, false);
+	if (ret < 0)
+		return ERR_PTR(ret);
+	return bh;
+}
+
+struct erofs_buffer_head *erofs_battach(struct erofs_buffer_head *bh,
+					int type, int size)
+{
+	struct erofs_buffer_block *const bb = bh->block;
+	struct erofs_buffer_head *nbh;
+	unsigned int alignsize;
+	int ret = get_alignsize(type, &type);
+
+	if (ret < 0)
+		return ERR_PTR(ret);
+	alignsize = ret;
+
+	/* should be the tail bh in the corresponding buffer block */
+	if (bh->list.next != &bb->buffers.list)
+		return ERR_PTR(-EINVAL);
+
+	nbh = malloc(sizeof(*nbh));
+	if (!nbh)
+		return ERR_PTR(-ENOMEM);
+
+	ret = __erofs_battach(bb, nbh, size, alignsize, 0, false);
+	if (ret < 0) {
+		free(nbh);
+		return ERR_PTR(ret);
+	}
+	return nbh;
+
+}
+
+static erofs_blk_t __erofs_mapbh(struct erofs_buffer_block *bb)
+{
+	erofs_blk_t blkaddr;
+
+	if (bb->blkaddr == NULL_ADDR)
+		bb->blkaddr = tail_blkaddr;
+
+	blkaddr = bb->blkaddr + BLK_ROUND_UP(bb->buffers.off);
+	if (blkaddr > tail_blkaddr)
+		tail_blkaddr = blkaddr;
+
+	return blkaddr;
+}
+
+erofs_blk_t erofs_mapbh(struct erofs_buffer_block *bb, bool end)
+{
+	struct erofs_buffer_block *t, *nt;
+
+	if (!bb || bb->blkaddr == NULL_ADDR) {
+		list_for_each_entry_safe(t, nt, &blkh.list, list) {
+			if (!end && (t == bb || nt == &blkh))
+				break;
+			(void)__erofs_mapbh(t);
+			if (end && t == bb)
+				break;
+		}
+	}
+	return tail_blkaddr;
+}
+
+bool erofs_bflush(struct erofs_buffer_block *bb)
+{
+	static const char zero[EROFS_BLKSIZ] = {0};
+	struct erofs_buffer_block *p, *n;
+	erofs_blk_t blkaddr;
+
+	list_for_each_entry_safe(p, n, &blkh.list, list) {
+		struct erofs_buffer_head *bh, *nbh;
+		unsigned int padding;
+		bool skip = false;
+
+		if (p == bb)
+			break;
+
+		/* check if the buffer block can flush */
+		list_for_each_entry(bh, &p->buffers.list, list)
+			if (bh->op->preflush && !bh->op->preflush(bh))
+				return false;
+
+		blkaddr = __erofs_mapbh(p);
+
+		list_for_each_entry_safe(bh, nbh, &p->buffers.list, list) {
+			/* flush and remove bh */
+			if (!bh->op->flush(bh))
+				skip = true;
+		}
+
+		if (skip)
+			continue;
+
+		padding = EROFS_BLKSIZ - p->buffers.off % EROFS_BLKSIZ;
+		if (padding != EROFS_BLKSIZ)
+			dev_write(zero, blknr_to_addr(blkaddr) - padding,
+				  padding);
+
+		DBG_BUGON(!list_empty(&p->buffers.list));
+
+		erofs_dbg("block %u to %u flushed", p->blkaddr, blkaddr - 1);
+
+		list_del(&p->list);
+		free(p);
+	}
+	return true;
+}
+
+void erofs_bdrop(struct erofs_buffer_head *bh, bool tryrevoke)
+{
+	struct erofs_buffer_block *const bb = bh->block;
+	const erofs_blk_t blkaddr = bh->block->blkaddr;
+	bool rollback = false;
+
+	/* tail_blkaddr could be rolled back after revoking all bhs */
+	if (tryrevoke && blkaddr != NULL_ADDR &&
+	    tail_blkaddr == blkaddr + BLK_ROUND_UP(bb->buffers.off))
+		rollback = true;
+
+	bh->op = &erofs_drop_directly_bhops;
+	erofs_bh_flush_generic_end(bh);
+
+	if (!list_empty(&bb->buffers.list))
+		return;
+
+	list_del(&bb->list);
+	free(bb);
+
+	if (rollback)
+		tail_blkaddr = blkaddr;
+}
+
-- 
2.17.1

^ permalink raw reply related	[flat|nested] 14+ messages in thread
* [PATCH 06/13] erofs-utils: introduce inode operations
  2019-05-31  0:50 [PATCH 00/13] erofs_utils: new mkfs framework Gao Xiang
                   ` (4 preceding siblings ...)
  2019-05-31  0:50 ` [PATCH 05/13] erofs-utils: introduce buffer cache Gao Xiang
@ 2019-05-31  0:50 ` Gao Xiang
  2019-05-31  0:50 ` [PATCH 07/13] erofs-utils: introduce mkfs support Gao Xiang
                   ` (6 subsequent siblings)
  12 siblings, 0 replies; 14+ messages in thread
From: Gao Xiang @ 2019-05-31  0:50 UTC (permalink / raw)


From: Gao Xiang <gaoxiang25@huawei.com>

This patch adds core inode and dentry operations
to build the target image.

Signed-off-by: Li Guifu <bluce.liguifu at huawei.com>
[ Gao Xiang: with heavy changes. ]
Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 include/erofs/inode.h    |  21 ++
 include/erofs/internal.h |  10 +-
 lib/Makefile.am          |   2 +-
 lib/inode.c              | 752 +++++++++++++++++++++++++++++++++++++++
 4 files changed, 782 insertions(+), 3 deletions(-)
 create mode 100644 include/erofs/inode.h
 create mode 100644 lib/inode.c

diff --git a/include/erofs/inode.h b/include/erofs/inode.h
new file mode 100644
index 0000000..43aee93
--- /dev/null
+++ b/include/erofs/inode.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/inode.h
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ * with heavy changes by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#ifndef __EROFS_INODE_H
+#define __EROFS_INODE_H
+
+#include "erofs/internal.h"
+
+void erofs_inode_manager_init(void);
+unsigned int erofs_iput(struct erofs_inode *inode);
+erofs_nid_t erofs_lookupnid(struct erofs_inode *inode);
+struct erofs_inode *erofs_mkfs_build_tree_from_path(struct erofs_inode *parent,
+						    const char *path);
+
+#endif
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 238a076..dfa6173 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -82,8 +82,6 @@ struct erofs_inode {
 	char i_srcpath[PATH_MAX + 1];
 
 	unsigned char data_mapping_mode;
-	bool compression_disabled;
-
 	unsigned char inode_isize;
 	/* inline tail-end packing size */
 	unsigned short idata_size;
@@ -111,6 +109,14 @@ struct erofs_dentry {
 	};
 };
 
+static inline bool is_dot_dotdot(const char *name)
+{
+	if (name[0] != '.')
+		return false;
+
+	return name[1] == '\0' || (name[1] == '.' && name[2] == '\0');
+}
+
 #include <stdio.h>
 #include <string.h>
 
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 8508660..5257d71 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -2,6 +2,6 @@
 # Makefile.am
 
 noinst_LTLIBRARIES = liberofs.la
-liberofs_la_SOURCES = config.c io.c cache.c
+liberofs_la_SOURCES = config.c io.c cache.c inode.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 
diff --git a/lib/inode.c b/lib/inode.c
new file mode 100644
index 0000000..d3e5ed8
--- /dev/null
+++ b/lib/inode.c
@@ -0,0 +1,752 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs_utils/lib/inode.c
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ * with heavy changes by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#define _GNU_SOURCE
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include "erofs/print.h"
+#include "erofs/inode.h"
+#include "erofs/cache.h"
+#include "erofs/io.h"
+
+struct erofs_sb_info sbi;
+
+#define S_SHIFT                 12
+static unsigned char erofs_type_by_mode[S_IFMT >> S_SHIFT] = {
+	[S_IFREG >> S_SHIFT]  = EROFS_FT_REG_FILE,
+	[S_IFDIR >> S_SHIFT]  = EROFS_FT_DIR,
+	[S_IFCHR >> S_SHIFT]  = EROFS_FT_CHRDEV,
+	[S_IFBLK >> S_SHIFT]  = EROFS_FT_BLKDEV,
+	[S_IFIFO >> S_SHIFT]  = EROFS_FT_FIFO,
+	[S_IFSOCK >> S_SHIFT] = EROFS_FT_SOCK,
+	[S_IFLNK >> S_SHIFT]  = EROFS_FT_SYMLINK,
+};
+
+#define NR_INODE_HASHTABLE	64
+
+struct list_head inode_hashtable[NR_INODE_HASHTABLE];
+
+void erofs_inode_manager_init(void)
+{
+	unsigned int i;
+
+	for (i = 0; i < NR_INODE_HASHTABLE; ++i)
+		init_list_head(&inode_hashtable[i]);
+}
+
+static struct erofs_inode *erofs_igrab(struct erofs_inode *inode)
+{
+	++inode->i_count;
+	return inode;
+}
+
+/* get the inode from the (source) inode # */
+struct erofs_inode *erofs_iget(ino_t ino)
+{
+	struct list_head *head =
+		&inode_hashtable[ino % NR_INODE_HASHTABLE];
+	struct erofs_inode *inode;
+
+	list_for_each_entry(inode, head, i_hash)
+		if (inode->i_ino[1] == ino)
+			return erofs_igrab(inode);
+	return NULL;
+}
+
+struct erofs_inode *erofs_iget_by_nid(erofs_nid_t nid)
+{
+	struct list_head *head =
+		&inode_hashtable[nid % NR_INODE_HASHTABLE];
+	struct erofs_inode *inode;
+
+	list_for_each_entry(inode, head, i_hash)
+		if (inode->nid == nid)
+			return erofs_igrab(inode);
+	return NULL;
+}
+
+unsigned int erofs_iput(struct erofs_inode *inode)
+{
+	struct erofs_dentry *d, *t;
+
+	if (inode->i_count > 1)
+		return --inode->i_count;
+
+	list_for_each_entry_safe(d, t, &inode->i_subdirs, d_child)
+		free(d);
+
+	list_del(&inode->i_hash);
+	free(inode);
+	return 0;
+}
+
+static int dentry_add_sorted(struct erofs_dentry *d, struct list_head *head)
+{
+	struct list_head *pos;
+
+	list_for_each(pos, head) {
+		struct erofs_dentry *d2 =
+			container_of(pos, struct erofs_dentry, d_child);
+
+		if (strcmp(d->name, d2->name) < 0)
+			break;
+	}
+	list_add_tail(&d->d_child, pos);
+	return 0;
+}
+
+struct erofs_dentry *erofs_d_alloc(struct erofs_inode *parent,
+				   const char *name)
+{
+	struct erofs_dentry *d = malloc(sizeof(*d));
+
+	if (!d)
+		return ERR_PTR(-ENOMEM);
+
+	strncpy(d->name, name, EROFS_NAME_LEN - 1);
+	d->name[EROFS_NAME_LEN - 1] = '\0';
+
+	dentry_add_sorted(d, &parent->i_subdirs);
+	return d;
+}
+
+/* allocate main data for a inode */
+static int __allocate_inode_bh_data(struct erofs_inode *inode,
+				    unsigned long nblocks)
+{
+	struct erofs_buffer_head *bh;
+	int ret;
+
+	if (!nblocks) {
+		inode->bh_data = NULL;
+		/* it has only tail-end inlined data */
+		inode->u.i_blkaddr = NULL_ADDR;
+		return 0;
+	}
+
+	/* allocate main data buffer */
+	bh = erofs_balloc(DATA, blknr_to_addr(nblocks), 0, 0);
+	if (IS_ERR(bh))
+		return PTR_ERR(bh);
+
+	bh->op = &erofs_skip_write_bhops;
+	inode->bh_data = bh;
+
+	/* get blkaddr of the bh */
+	ret = erofs_mapbh(bh->block, true);
+	DBG_BUGON(ret < 0);
+
+	/* write blocks except for the tail-end block */
+	inode->u.i_blkaddr = bh->block->blkaddr;
+	return 0;
+}
+
+int erofs_prepare_dir_file(struct erofs_inode *dir)
+{
+	struct erofs_dentry *d;
+	unsigned int d_size;
+	int ret;
+
+	/* dot is pointed to the current dir inode */
+	d = erofs_d_alloc(dir, ".");
+	d->inode = erofs_igrab(dir);
+	d->type = EROFS_FT_DIR;
+
+	/* dotdot is pointed to the parent dir */
+	d = erofs_d_alloc(dir, "..");
+	d->inode = erofs_igrab(dir->i_parent);
+	d->type = EROFS_FT_DIR;
+
+	/* let's calculate dir size */
+	d_size = 0;
+	list_for_each_entry(d, &dir->i_subdirs, d_child) {
+		int len = strlen(d->name) + sizeof(struct erofs_dirent);
+
+		if (d_size % EROFS_BLKSIZ + len > EROFS_BLKSIZ)
+			d_size = round_up(d_size, EROFS_BLKSIZ);
+		d_size += len;
+	}
+	dir->i_size = d_size;
+
+	/* no compression for all dirs */
+	dir->data_mapping_mode = EROFS_INODE_LAYOUT_INLINE;
+
+	/* allocate dir main data */
+	ret = __allocate_inode_bh_data(dir, erofs_blknr(d_size));
+	if (ret)
+		return ret;
+
+	/* it will be used in erofs_prepare_inode_buffer */
+	dir->idata_size = d_size % EROFS_BLKSIZ;
+	return 0;
+}
+
+static void fill_dirblock(char *buf, unsigned int size, unsigned int q,
+			  struct erofs_dentry *head, struct erofs_dentry *end)
+{
+	unsigned int p = 0;
+
+	/* write out all erofs_dirents + filenames */
+	while (head != end) {
+		const unsigned int namelen = strlen(head->name);
+		struct erofs_dirent d = {
+			.nid = cpu_to_le64(head->nid),
+			.nameoff = cpu_to_le16(q),
+			.file_type = head->type,
+		};
+
+		memcpy(buf + p, &d, sizeof(d));
+		memcpy(buf + q, head->name, namelen);
+		p += sizeof(d);
+		q += namelen;
+
+		head = list_next_entry(head, d_child);
+	}
+	memset(buf + q, 0, size - q);
+}
+
+int erofs_write_dir_file(struct erofs_inode *dir)
+{
+	struct erofs_dentry *head = list_first_entry(&dir->i_subdirs,
+						     struct erofs_dentry,
+						     d_child);
+	struct erofs_dentry *d;
+	int ret;
+	unsigned int q, used, blkno;
+
+	q = used = blkno = 0;
+
+	list_for_each_entry(d, &dir->i_subdirs, d_child) {
+		const unsigned int len = strlen(d->name) +
+			sizeof(struct erofs_dirent);
+
+		if (used + len > EROFS_BLKSIZ) {
+			char buf[EROFS_BLKSIZ];
+
+			fill_dirblock(buf, EROFS_BLKSIZ, q, head, d);
+			ret = blk_write(buf, dir->u.i_blkaddr + blkno, 1);
+			if (ret)
+				return ret;
+
+			head = d;
+			q = used = 0;
+			++blkno;
+		}
+		used += len;
+		q += sizeof(struct erofs_dirent);
+	}
+	DBG_BUGON(used != dir->i_size % EROFS_BLKSIZ);
+	if (used) {
+		/* fill tail-end dir block */
+		dir->idata = malloc(used);
+		fill_dirblock(dir->idata, dir->idata_size, q, head, d);
+	}
+	return 0;
+}
+
+int erofs_write_file_from_buffer(struct erofs_inode *inode, char *buf)
+{
+	const unsigned int nblocks = erofs_blknr(inode->i_size);
+	int ret;
+
+	inode->data_mapping_mode = EROFS_INODE_LAYOUT_INLINE;
+
+	ret = __allocate_inode_bh_data(inode, nblocks);
+	if (ret)
+		return ret;
+
+	if (nblocks)
+		blk_write(buf, inode->u.i_blkaddr, nblocks);
+	inode->idata_size = inode->i_size % EROFS_BLKSIZ;
+	if (inode->idata_size) {
+		inode->idata = malloc(inode->idata_size);
+		memcpy(inode->idata, buf + blknr_to_addr(nblocks),
+		       inode->idata_size);
+	}
+	return 0;
+}
+
+/* rules to decide whether a file could be compressed or not */
+static bool erofs_file_is_compressible(struct erofs_inode *inode)
+{
+	return true;
+}
+
+int erofs_write_file(struct erofs_inode *inode)
+{
+	unsigned int nblocks, i;
+	int ret, fd;
+
+	if (erofs_file_is_compressible(inode)) {
+		/* to be implemented */
+	}
+
+	/* fallback to all data uncompressed */
+	inode->data_mapping_mode = EROFS_INODE_LAYOUT_INLINE;
+	nblocks = inode->i_size / EROFS_BLKSIZ;
+
+	ret = __allocate_inode_bh_data(inode, nblocks);
+	if (ret)
+		return ret;
+
+	fd = open(inode->i_srcpath, O_RDONLY | O_BINARY);
+	if (fd < 0)
+		return -errno;
+
+	for (i = 0; i < nblocks; ++i) {
+		char buf[EROFS_BLKSIZ];
+
+		ret = read(fd, buf, EROFS_BLKSIZ);
+		if (ret != EROFS_BLKSIZ) {
+			if (ret < 0)
+				goto fail;
+			close(fd);
+			return -EAGAIN;
+		}
+
+		ret = blk_write(buf, inode->u.i_blkaddr + i, 1);
+		if (ret)
+			goto fail;
+	}
+
+	/* read the tail-end data */
+	inode->idata_size = inode->i_size % EROFS_BLKSIZ;
+	if (inode->idata_size) {
+		inode->idata = malloc(inode->idata_size);
+
+		ret = read(fd, inode->idata, inode->idata_size);
+		if (ret < inode->idata_size) {
+			close(fd);
+			return -EIO;
+		}
+	}
+	close(fd);
+	return 0;
+fail:
+	ret = -errno;
+	close(fd);
+	return ret;
+}
+
+static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)
+{
+	struct erofs_inode *const inode = bh->fsprivate;
+	const erofs_off_t off = erofs_btell(bh, false);
+
+	/* let's support v1 currently */
+	struct erofs_inode_v1 v1 = {0};
+	int ret;
+
+	v1.i_advise = cpu_to_le16(0 | (inode->data_mapping_mode << 1));
+	v1.i_mode = cpu_to_le16(inode->i_mode);
+	v1.i_nlink = cpu_to_le16(inode->i_nlink);
+	v1.i_size = cpu_to_le32((u32)inode->i_size);
+
+	v1.i_ino = cpu_to_le32(inode->i_ino[0]);
+
+	v1.i_uid = cpu_to_le16((u16)inode->i_uid);
+	v1.i_gid = cpu_to_le16((u16)inode->i_gid);
+
+	switch ((inode->i_mode) >> S_SHIFT) {
+	case S_IFCHR:
+	case S_IFBLK:
+	case S_IFIFO:
+	case S_IFSOCK:
+		v1.i_u.rdev = cpu_to_le32(inode->u.i_rdev);
+		break;
+
+	default:
+		if (inode->data_mapping_mode == EROFS_INODE_LAYOUT_COMPRESSION)
+			v1.i_u.compressed_blocks =
+				cpu_to_le32(inode->u.i_blocks);
+		else
+			v1.i_u.raw_blkaddr =
+				cpu_to_le32(inode->u.i_blkaddr);
+		break;
+	}
+	v1.i_checksum = 0;
+
+	ret = dev_write(&v1, off, sizeof(struct erofs_inode_v1));
+	if (ret)
+		return false;
+
+	erofs_iput(inode);
+	return erofs_bh_flush_generic_end(bh);
+}
+
+static struct erofs_bhops erofs_write_inode_bhops = {
+	.flush = erofs_bh_flush_write_inode,
+};
+
+int erofs_prepare_inode_buffer(struct erofs_inode *inode)
+{
+	unsigned int inodesize;
+	struct erofs_buffer_head *bh, *ibh;
+
+	DBG_BUGON(inode->bh || inode->bh_inline);
+
+	inodesize = inode->inode_isize + inode->xattr_isize +
+		    inode->extent_isize;
+
+	if (inode->data_mapping_mode == EROFS_INODE_LAYOUT_COMPRESSION)
+		goto noinline;
+
+	/*
+	 * if the file size is block-aligned for uncompressed files,
+	 * should use EROFS_INODE_LAYOUT_PLAIN data mapping mode.
+	 */
+	if (!inode->idata_size)
+		inode->data_mapping_mode = EROFS_INODE_LAYOUT_PLAIN;
+
+	bh = erofs_balloc(INODE, inodesize, 0, inode->idata_size);
+	if (bh == ERR_PTR(-ENOSPC)) {
+		inode->data_mapping_mode = EROFS_INODE_LAYOUT_PLAIN;
+noinline:
+		bh = erofs_balloc(INODE, inodesize, 0, 0);
+		if (IS_ERR(bh))
+			return PTR_ERR(bh);
+	} else if (IS_ERR(bh)) {
+		return PTR_ERR(bh);
+	} else if (inode->idata_size) {
+		inode->data_mapping_mode = EROFS_INODE_LAYOUT_INLINE;
+
+		/* allocate inline buffer */
+		ibh = erofs_battach(bh, META, inode->idata_size);
+		if (IS_ERR(ibh))
+			return PTR_ERR(ibh);
+
+		ibh->op = &erofs_skip_write_bhops;
+		inode->bh_inline = ibh;
+	}
+
+	bh->fsprivate = erofs_igrab(inode);
+	bh->op = &erofs_write_inode_bhops;
+	inode->bh = bh;
+	return 0;
+}
+
+static bool erofs_bh_flush_write_inline(struct erofs_buffer_head *bh)
+{
+	struct erofs_inode *const inode = bh->fsprivate;
+	const erofs_off_t off = erofs_btell(bh, false);
+	int ret;
+
+	ret = dev_write(inode->idata, off, inode->idata_size);
+	if (ret)
+		return false;
+
+	inode->idata_size = 0;
+	free(inode->idata);
+	inode->idata = NULL;
+
+	erofs_iput(inode);
+	return erofs_bh_flush_generic_end(bh);
+}
+
+static struct erofs_bhops erofs_write_inline_bhops = {
+	.flush = erofs_bh_flush_write_inline,
+};
+
+int erofs_write_tail_end(struct erofs_inode *inode)
+{
+	struct erofs_buffer_head *bh, *ibh;
+
+	bh = inode->bh_data;
+
+	if (!inode->idata_size)
+		goto out;
+
+	/* have enough room to inline data */
+	if (inode->bh_inline) {
+		ibh = inode->bh_inline;
+
+		ibh->fsprivate = erofs_igrab(inode);
+		ibh->op = &erofs_write_inline_bhops;
+	} else {
+		int ret;
+		erofs_off_t off;
+
+		if (bh) {
+			/* expend a block (should be successful) */
+			ret = erofs_bh_balloon(bh, EROFS_BLKSIZ);
+			DBG_BUGON(ret);
+
+			erofs_mapbh(bh->block, true);
+			off = erofs_btell(bh, true) - EROFS_BLKSIZ;
+		} else {
+			bh = erofs_balloc(DATA, EROFS_BLKSIZ, 0, 0);
+			if (IS_ERR(bh))
+				return PTR_ERR(bh);
+
+			bh->op = &erofs_skip_write_bhops;
+			erofs_mapbh(bh->block, true);
+			off = erofs_btell(bh, false);
+			inode->u.i_blkaddr = erofs_blknr(off);
+			inode->bh_data = bh;
+		}
+
+		ret = dev_write(inode->idata, off, inode->idata_size);
+		if (ret)
+			return ret;
+
+		inode->idata_size = 0;
+		free(inode->idata);
+		inode->idata = NULL;
+	}
+out:
+	/* now bh_data can drop directly */
+	if (bh) {
+		/*
+		 * Don't leave DATA buffers which were written in the global
+		 * buffer list. It will make balloc() slowly.
+		 */
+#if 0
+		bh->op = &erofs_drop_directly_bhops;
+#else
+		erofs_bdrop(bh, false);
+#endif
+		inode->bh_data = NULL;
+	}
+	return 0;
+}
+
+int erofs_fill_inode(struct erofs_inode *inode,
+		     struct stat64 *st,
+		     const char *path)
+{
+	inode->i_mode = st->st_mode;
+	inode->i_uid = st->st_uid;
+	inode->i_gid = st->st_gid;
+	inode->i_nlink = 1;	/* fix up later if needed */
+
+	if (!S_ISDIR(inode->i_mode))
+		inode->i_size = st->st_size;
+	else
+		inode->i_size = 0;
+
+	strncpy(inode->i_srcpath, path, sizeof(inode->i_srcpath) - 1);
+	inode->i_srcpath[sizeof(inode->i_srcpath) - 1] = '\0';
+
+	inode->i_ino[1] = st->st_ino;
+	inode->inode_isize = sizeof(struct erofs_inode_v1);
+
+	list_add(&inode->i_hash,
+		 &inode_hashtable[st->st_ino % NR_INODE_HASHTABLE]);
+	return 0;
+}
+
+struct erofs_inode *erofs_new_inode(void)
+{
+	static unsigned int counter;
+	struct erofs_inode *inode;
+
+	inode = malloc(sizeof(struct erofs_inode));
+	if (!inode)
+		return ERR_PTR(-ENOMEM);
+
+	inode->i_parent = NULL;	/* also used to indicate a new inode */
+
+	inode->i_ino[0] = counter++;	/* inode serial number */
+	inode->i_count = 1;
+
+	init_list_head(&inode->i_subdirs);
+	inode->xattr_isize = 0;
+	inode->extent_isize = 0;
+
+	inode->bh = inode->bh_inline = inode->bh_data = NULL;
+	inode->idata = NULL;
+	return inode;
+}
+
+/* get the inode from the (source) path */
+struct erofs_inode *erofs_iget_from_path(const char *path, bool is_src)
+{
+	struct stat64 st;
+	struct erofs_inode *inode;
+	int ret;
+
+	/* currently, only source path is supported */
+	if (!is_src)
+		return ERR_PTR(-EINVAL);
+
+	ret = lstat64(path, &st);
+	if (ret)
+		return ERR_PTR(-errno);
+
+	inode = erofs_iget(st.st_ino);
+	if (inode)
+		return inode;
+
+	/* cannot find in the inode cache */
+	inode = erofs_new_inode();
+	if (IS_ERR(inode))
+		return inode;
+
+	ret = erofs_fill_inode(inode, &st, path);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return inode;
+}
+
+erofs_nid_t erofs_lookupnid(struct erofs_inode *inode)
+{
+	struct erofs_buffer_block *const bb = inode->bh->block;
+	erofs_off_t off, meta_offset;
+
+	if (!inode->bh)
+		return inode->nid;
+
+	erofs_mapbh(bb, true);
+	off = erofs_btell(inode->bh, false);
+	meta_offset = blknr_to_addr(sbi.meta_blkaddr);
+
+	if (off - blknr_to_addr(sbi.meta_blkaddr) > 0xffff) {
+		if (IS_ROOT(inode)) {
+			meta_offset = round_up(off - 0xffff, EROFS_BLKSIZ);
+			sbi.meta_blkaddr = meta_offset / EROFS_BLKSIZ;
+		}
+	}
+	return inode->nid = (off - meta_offset) >> EROFS_ISLOTBITS;
+}
+
+void erofs_d_invalidate(struct erofs_dentry *d)
+{
+	struct erofs_inode *const inode = d->inode;
+
+	d->nid = erofs_lookupnid(inode);
+	erofs_iput(inode);
+}
+
+struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir)
+{
+	int ret;
+	DIR *_dir;
+	struct dirent *dp;
+	struct erofs_dentry *d;
+
+	if (!S_ISDIR(dir->i_mode)) {
+		if (S_ISLNK(dir->i_mode)) {
+			char *const symlink = malloc(dir->i_size);
+
+			ret = readlink(dir->i_srcpath, symlink, dir->i_size);
+			if (ret < 0)
+				return ERR_PTR(-errno);
+
+			erofs_write_file_from_buffer(dir, symlink);
+			free(symlink);
+		} else {
+			erofs_write_file(dir);
+		}
+
+		erofs_prepare_inode_buffer(dir);
+		erofs_write_tail_end(dir);
+		return dir;
+	}
+
+	_dir = opendir(dir->i_srcpath);
+	if (!_dir) {
+		erofs_err("%s, failed to opendir at %s: %s",
+			  __func__, dir->i_srcpath, erofs_strerror(errno));
+		return ERR_PTR(-errno);
+	}
+
+	while (1) {
+		/*
+		 * set errno to 0 before calling readdir() in order to
+		 * distinguish end of stream and from an error.
+		 */
+		errno = 0;
+		dp = readdir(_dir);
+		if (!dp)
+			break;
+
+		if (is_dot_dotdot(dp->d_name) ||
+		    !strncmp(dp->d_name, "lost+found", strlen("lost+found")))
+			continue;
+
+		d = erofs_d_alloc(dir, dp->d_name);
+		if (IS_ERR(d)) {
+			ret = PTR_ERR(d);
+			goto err_closedir;
+		}
+	}
+
+	if (errno) {
+		ret = -errno;
+		goto err_closedir;
+	}
+	closedir(_dir);
+
+	erofs_prepare_dir_file(dir);
+	erofs_prepare_inode_buffer(dir);
+
+	list_for_each_entry(d, &dir->i_subdirs, d_child) {
+		char buf[PATH_MAX];
+
+		if (is_dot_dotdot(d->name)) {
+			erofs_d_invalidate(d);
+			continue;
+		}
+
+		ret = snprintf(buf, PATH_MAX, "%s/%s",
+			       dir->i_srcpath, d->name);
+		if (ret < 0 || ret >= PATH_MAX) {
+			/* ignore the too long path */
+			goto fail;
+		}
+
+		d->inode = erofs_mkfs_build_tree_from_path(dir, buf);
+		if (IS_ERR(d->inode)) {
+fail:
+			d->inode = NULL;
+			d->type = EROFS_FT_UNKNOWN;
+			continue;
+		}
+
+		d->type = erofs_type_by_mode[d->inode->i_mode >> S_SHIFT];
+		erofs_d_invalidate(d);
+		erofs_info("add file %s/%s (nid %lu, type %d)",
+			   dir->i_srcpath, d->name, d->nid, d->type);
+	}
+	erofs_write_dir_file(dir);
+	erofs_write_tail_end(dir);
+	return dir;
+
+err_closedir:
+	closedir(_dir);
+	return ERR_PTR(ret);
+}
+
+struct erofs_inode *erofs_mkfs_build_tree_from_path(struct erofs_inode *parent,
+						    const char *path)
+{
+	struct erofs_inode *const inode = erofs_iget_from_path(path, true);
+
+	if (IS_ERR(inode))
+		return inode;
+
+	/* a hardlink to the existed inode */
+	if (inode->i_parent) {
+		++inode->i_nlink;
+		return inode;
+	}
+
+	/* a completely new inode is found */
+	if (parent)
+		inode->i_parent = parent;
+	else
+		inode->i_parent = inode;	/* rootdir mark */
+
+	return erofs_mkfs_build_tree(inode);
+}
+
-- 
2.17.1

^ permalink raw reply related	[flat|nested] 14+ messages in thread
* [PATCH 07/13] erofs-utils: introduce mkfs support
  2019-05-31  0:50 [PATCH 00/13] erofs_utils: new mkfs framework Gao Xiang
                   ` (5 preceding siblings ...)
  2019-05-31  0:50 ` [PATCH 06/13] erofs-utils: introduce inode operations Gao Xiang
@ 2019-05-31  0:50 ` Gao Xiang
  2019-05-31  0:50 ` [PATCH 08/13] erofs-utils: introduce generic compression framework Gao Xiang
                   ` (5 subsequent siblings)
  12 siblings, 0 replies; 14+ messages in thread
From: Gao Xiang @ 2019-05-31  0:50 UTC (permalink / raw)


From: Li Guifu <bluce.liguifu@huawei.com>

This patch adds mkfs support to erofs-utils, and
it's able to build uncompressed images at the moment.

Signed-off-by: Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 Makefile.am      |   2 +-
 configure.ac     |   3 +-
 mkfs/Makefile.am |   9 +++
 mkfs/main.c      | 179 +++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 191 insertions(+), 2 deletions(-)
 create mode 100644 mkfs/Makefile.am
 create mode 100644 mkfs/main.c

diff --git a/Makefile.am b/Makefile.am
index ee5fd92..d94ab73 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -3,4 +3,4 @@
 
 ACLOCAL_AMFLAGS = -I m4
 
-SUBDIRS=lib
+SUBDIRS=lib mkfs
diff --git a/configure.ac b/configure.ac
index 9c6d8bb..49f1a7d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -103,5 +103,6 @@ AC_CHECK_DECL(lseek64,[AC_DEFINE(HAVE_LSEEK64_PROTOTYPE, 1,
 AC_CHECK_FUNCS([gettimeofday memset realpath strdup strerror strrchr strtoull])
 
 AC_CONFIG_FILES([Makefile
-		 lib/Makefile])
+		 lib/Makefile
+		 mkfs/Makefile])
 AC_OUTPUT
diff --git a/mkfs/Makefile.am b/mkfs/Makefile.am
new file mode 100644
index 0000000..257f864
--- /dev/null
+++ b/mkfs/Makefile.am
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0+
+# Makefile.am
+
+AUTOMAKE_OPTIONS = foreign
+bin_PROGRAMS     = mkfs.erofs
+mkfs_erofs_SOURCES = main.c
+mkfs_erofs_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
+mkfs_erofs_LDADD = $(top_builddir)/lib/liberofs.la
+
diff --git a/mkfs/main.c b/mkfs/main.c
new file mode 100644
index 0000000..1ed15d2
--- /dev/null
+++ b/mkfs/main.c
@@ -0,0 +1,179 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * mkfs/main.c
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#define _GNU_SOURCE
+#include <time.h>
+#include <sys/time.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <libgen.h>
+#include "erofs/config.h"
+#include "erofs/print.h"
+#include "erofs/cache.h"
+#include "erofs/inode.h"
+#include "erofs/io.h"
+
+#define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
+
+static void usage(char *execpath)
+{
+	fprintf(stderr, "%s %s\n", basename(execpath), cfg.c_version);
+	fprintf(stderr, "usage: [options] FILE DIRECTORY\n\n");
+	fprintf(stderr, "Generate erofs image from DIRECTORY to FILE, and [options] are:\n");
+	fprintf(stderr, " -d#      set output message level to # (maximum 9)\n");
+}
+
+u64 parse_num_from_str(const char *str)
+{
+	u64 num      = 0;
+	char *endptr = NULL;
+
+	num = strtoull(str, &endptr, 10);
+	BUG_ON(num == ULLONG_MAX);
+	return num;
+}
+
+static int mkfs_parse_options_cfg(int argc, char *argv[])
+{
+	int opt, i;
+
+	while ((opt = getopt(argc, argv, "d:z:")) != -1) {
+		switch (opt) {
+		case 'd':
+			cfg.c_dbg_lvl = parse_num_from_str(optarg);
+			break;
+
+		default: /* '?' */
+			return -EINVAL;
+		}
+	}
+
+	if (optind >= argc)
+		return -EINVAL;
+
+	cfg.c_img_path = strdup(argv[optind++]);
+	if (!cfg.c_img_path)
+		return -ENOMEM;
+
+	if (optind > argc) {
+		erofs_err("Source directory is missing");
+		return -EINVAL;
+	}
+
+	cfg.c_src_path = realpath(argv[optind++], NULL);
+	if (!cfg.c_src_path) {
+		erofs_err("Failed to parse source directory: %s",
+			  erofs_strerror(-errno));
+		return -ENOENT;
+	}
+
+	if (optind < argc) {
+		erofs_err("Unexpected argument: %s\n", argv[optind]);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
+				  erofs_nid_t root_nid)
+{
+	struct erofs_super_block sb = {
+		.magic     = cpu_to_le32(EROFS_SUPER_MAGIC_V1),
+		.blkszbits = LOG_BLOCK_SIZE,
+		.inos   = 0,
+		.blocks = 0,
+		.meta_blkaddr  = sbi.meta_blkaddr,
+		.xattr_blkaddr = 0,
+	};
+	const unsigned int sb_blksize =
+		round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
+	char *buf;
+	struct timeval t;
+
+	if (!gettimeofday(&t, NULL)) {
+		sb.build_time      = cpu_to_le64(t.tv_sec);
+		sb.build_time_nsec = cpu_to_le32(t.tv_usec);
+	}
+
+	sb.blocks       = cpu_to_le32(erofs_mapbh(NULL, true));
+	sb.root_nid     = cpu_to_le16(root_nid);
+
+	buf = calloc(sb_blksize, 1);
+	if (!buf) {
+		erofs_err("Failed to allocate memory for sb: %s",
+			  erofs_strerror(-errno));
+		return -ENOMEM;
+	}
+	memcpy(buf + EROFS_SUPER_OFFSET, &sb, sizeof(sb));
+
+	bh->fsprivate = buf;
+	bh->op = &erofs_buf_write_bhops;
+	return 0;
+}
+
+int main(int argc, char **argv)
+{
+	int err = 0;
+	struct erofs_buffer_head *sb_bh;
+	struct erofs_inode *root_inode;
+	erofs_nid_t root_nid;
+
+	erofs_init_configure();
+	err = mkfs_parse_options_cfg(argc, argv);
+	if (err) {
+		if (err == -EINVAL)
+			usage(argv[0]);
+		return 1;
+	}
+
+	err = dev_open(cfg.c_img_path);
+	if (err) {
+		usage(argv[0]);
+		return 1;
+	}
+
+	erofs_err("%s %s\n", basename(argv[0]), cfg.c_version);
+	erofs_show_config();
+
+	sb_bh = erofs_buffer_init();
+	err = erofs_bh_balloon(sb_bh, EROFS_SUPER_END);
+	if (err < 0) {
+		erofs_err("Failed to balloon erofs_super_block: %s",
+			  erofs_strerror(err));
+		goto exit;
+	}
+
+	erofs_inode_manager_init();
+
+	root_inode = erofs_mkfs_build_tree_from_path(NULL, cfg.c_src_path);
+	if (IS_ERR(root_inode)) {
+		err = PTR_ERR(root_inode);
+		goto exit;
+	}
+
+	root_nid = erofs_lookupnid(root_inode);
+	erofs_iput(root_inode);
+
+	err = erofs_mkfs_update_super_block(sb_bh, root_nid);
+	if (err)
+		goto exit;
+
+	/* flush all remaining buffers */
+	if (!erofs_bflush(NULL))
+		err = -EIO;
+exit:
+	dev_close();
+	erofs_exit_configure();
+
+	if (err) {
+		erofs_err("\tCould not format the device : %s\n",
+			  erofs_strerror(err));
+		return 1;
+	}
+	return err;
+}
-- 
2.17.1

^ permalink raw reply related	[flat|nested] 14+ messages in thread
* [PATCH 08/13] erofs-utils: introduce generic compression framework
  2019-05-31  0:50 [PATCH 00/13] erofs_utils: new mkfs framework Gao Xiang
                   ` (6 preceding siblings ...)
  2019-05-31  0:50 ` [PATCH 07/13] erofs-utils: introduce mkfs support Gao Xiang
@ 2019-05-31  0:50 ` Gao Xiang
  2019-05-31  0:50 ` [PATCH 09/13] erofs-utils: introduce lz4/lz4hc compression algorithm Gao Xiang
                   ` (4 subsequent siblings)
  12 siblings, 0 replies; 14+ messages in thread
From: Gao Xiang @ 2019-05-31  0:50 UTC (permalink / raw)


From: Gao Xiang <gaoxiang25@huawei.com>

This patch adds a new flexable compression framework
to user-space utilities, which is designed in order
to integrate more compression algorithms easily.

Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 lib/Makefile.am  |  2 +-
 lib/compressor.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++
 lib/compressor.h | 48 +++++++++++++++++++++++++++++++
 3 files changed, 123 insertions(+), 1 deletion(-)
 create mode 100644 lib/compressor.c
 create mode 100644 lib/compressor.h

diff --git a/lib/Makefile.am b/lib/Makefile.am
index 5257d71..64da708 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -2,6 +2,6 @@
 # Makefile.am
 
 noinst_LTLIBRARIES = liberofs.la
-liberofs_la_SOURCES = config.c io.c cache.c inode.c
+liberofs_la_SOURCES = config.c io.c cache.c inode.c compressor.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 
diff --git a/lib/compressor.c b/lib/compressor.c
new file mode 100644
index 0000000..cc97cfb
--- /dev/null
+++ b/lib/compressor.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/compressor.c
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#include "erofs/internal.h"
+#include "compressor.h"
+
+#define EROFS_CONFIG_COMPR_DEF_BOUNDARY		(128)
+
+int erofs_compress_destsize(struct erofs_compress *c,
+			    int compression_level,
+			    void *src,
+			    unsigned int *srcsize,
+			    void *dst,
+			    unsigned int dstsize)
+{
+	int ret;
+
+	DBG_BUGON(!c->alg);
+	if (!c->alg->compress_destsize)
+		return -ENOTSUP;
+
+	ret = c->alg->compress_destsize(c, compression_level,
+					src, srcsize, dst, dstsize);
+	if (ret)
+		return ret;
+
+	/* check if there is enough gains to compress */
+	if (*srcsize <= dstsize * c->compress_threshold / 100)
+		return -EAGAIN;
+	return 0;
+}
+
+int erofs_compressor_init(struct erofs_compress *c,
+			  char *alg_name)
+{
+	static struct erofs_compressor *compressors[] = {
+	};
+
+	int ret, i;
+
+	/* should be written in "minimum compression ratio * 100" */
+	c->compress_threshold = 100;
+
+	/* optimize for 4k size page */
+	c->destsize_alignsize = PAGE_SIZE;
+	c->destsize_redzone_begin = PAGE_SIZE - 16;
+	c->destsize_redzone_end = EROFS_CONFIG_COMPR_DEF_BOUNDARY;
+
+	ret = -EINVAL;
+
+	for (i = 0; i < ARRAY_SIZE(compressors); ++i) {
+		ret = compressors[i]->init(c, alg_name);
+		if (!ret) {
+			DBG_BUGON(!c->alg);
+			return 0;
+		}
+	}
+	return ret;
+}
+
+int erofs_compressor_exit(struct erofs_compress *c)
+{
+	DBG_BUGON(!c->alg);
+
+	if (c->alg->exit)
+		return c->alg->exit(c);
+	return 0;
+}
+
diff --git a/lib/compressor.h b/lib/compressor.h
new file mode 100644
index 0000000..8ad9d11
--- /dev/null
+++ b/lib/compressor.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs-utils/lib/compressor.h
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#ifndef __EROFS_LIB_COMPRESSOR_H
+#define __EROFS_LIB_COMPRESSOR_H
+
+#include "erofs/defs.h"
+
+struct erofs_compress;
+
+struct erofs_compressor {
+	int default_level;
+	int best_level;
+
+	int (*init)(struct erofs_compress *c, char *alg_name);
+	int (*exit)(struct erofs_compress *c);
+
+	int (*compress_destsize)(struct erofs_compress *c,
+				 int compress_level,
+				 void *src, unsigned int *srcsize,
+				 void *dst, unsigned int dstsize);
+};
+
+struct erofs_compress {
+	struct erofs_compressor *alg;
+
+	unsigned int compress_threshold;
+
+	/* *_destsize specific */
+	unsigned int destsize_alignsize;
+	unsigned int destsize_redzone_begin;
+	unsigned int destsize_redzone_end;
+};
+
+int erofs_compress_destsize(struct erofs_compress *c, int compression_level,
+			    void *src, unsigned int *srcsize,
+			    void *dst, unsigned int dstsize);
+
+int erofs_compressor_init(struct erofs_compress *c, char *alg_name);
+int erofs_compressor_exit(struct erofs_compress *c);
+
+#endif
+
-- 
2.17.1

^ permalink raw reply related	[flat|nested] 14+ messages in thread
* [PATCH 09/13] erofs-utils: introduce lz4/lz4hc compression algorithm
  2019-05-31  0:50 [PATCH 00/13] erofs_utils: new mkfs framework Gao Xiang
                   ` (7 preceding siblings ...)
  2019-05-31  0:50 ` [PATCH 08/13] erofs-utils: introduce generic compression framework Gao Xiang
@ 2019-05-31  0:50 ` Gao Xiang
  2019-05-31  0:50 ` [PATCH 10/13] erofs-utils: introduce compression for regular files Gao Xiang
                   ` (3 subsequent siblings)
  12 siblings, 0 replies; 14+ messages in thread
From: Gao Xiang @ 2019-05-31  0:50 UTC (permalink / raw)


From: Gao Xiang <gaoxiang25@huawei.com>

This patch adds lz4/lz4hc compression algorithms to
erofs-utils compression framework in order to enable
the fixed-output size compression.

Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 configure.ac           | 65 ++++++++++++++++++++++++++++++++++++++++++
 lib/Makefile.am        |  7 +++++
 lib/compressor.c       |  6 ++++
 lib/compressor.h       |  6 ++++
 lib/compressor_lz4.c   | 48 +++++++++++++++++++++++++++++++
 lib/compressor_lz4hc.c | 61 +++++++++++++++++++++++++++++++++++++++
 6 files changed, 193 insertions(+)
 create mode 100644 lib/compressor_lz4.c
 create mode 100644 lib/compressor_lz4hc.c

diff --git a/configure.ac b/configure.ac
index 49f1a7d..6cc0c60 100644
--- a/configure.ac
+++ b/configure.ac
@@ -50,7 +50,22 @@ AC_DEFUN([EROFS_UTILS_PARSE_DIRECTORY],
  fi
 ])
 
+AC_ARG_ENABLE(lz4,
+   [AS_HELP_STRING([--disable-lz4], [disable LZ4 compression support @<:@default=enabled@:>@])],
+   [enable_lz4="$enableval"], [enable_lz4="yes"])
+
 # Checks for libraries.
+# Use customized LZ4 library path when specified.
+AC_ARG_WITH(lz4-incdir,
+   [AS_HELP_STRING([--with-lz4-incdir=DIR], [LZ4 include directory])], [
+   EROFS_UTILS_PARSE_DIRECTORY(["$withval"],[withval])])
+
+AC_ARG_WITH(lz4-libdir,
+   [AS_HELP_STRING([--with-lz4-libdir=DIR], [LZ4 lib directory])], [
+   EROFS_UTILS_PARSE_DIRECTORY(["$withval"],[withval])])
+
+AC_ARG_VAR([LZ4_CFLAGS], [C compiler flags for lz4])
+AC_ARG_VAR([LZ4_LIBS], [linker flags for lz4])
 
 # Checks for header files.
 AC_CHECK_HEADERS(m4_flatten([
@@ -102,7 +117,57 @@ AC_CHECK_DECL(lseek64,[AC_DEFINE(HAVE_LSEEK64_PROTOTYPE, 1,
 # Checks for library functions.
 AC_CHECK_FUNCS([gettimeofday memset realpath strdup strerror strrchr strtoull])
 
+# Configure lz4
+test -z $LZ4_LIBS && LZ4_LIBS='-llz4'
+
+if test "x$enable_lz4" = "xyes"; then
+  test -z "${with_lz4_incdir}" || LZ4_CFLAGS="-I$with_lz4_incdir $LZ4_CFLAGS"
+  test -z "${with_lz4_libdir}" || LZ4_LIBS="-L$with_lz4_libdir $LZ4_LIBS"
+
+  saved_CPPFLAGS=${CPPFLAGS}
+  CPPFLAGS="${LZ4_CFLAGS} ${CFLAGS}"
+
+  AC_CHECK_HEADERS([lz4.h],[have_lz4h="yes"], [])
+
+  if test "x${have_lz4h}" = "xyes" ; then
+    saved_LDFLAGS=${LDFLAGS}
+    LDFLAGS="-L$with_lz4_libdir ${LDFLAGS}"
+    AC_CHECK_LIB(lz4, LZ4_compress_destSize, [
+      have_lz4="yes"
+      have_lz4hc="yes"
+      AC_CHECK_LIB(lz4, LZ4_compress_HC_destSize, [], [
+        AC_CHECK_DECL(LZ4_compress_HC_destSize, [lz4_force_static="yes"],
+          [have_lz4hc="no"], [[
+#define LZ4_HC_STATIC_LINKING_ONLY (1)
+#include <lz4hc.h>
+        ]])
+      ])
+    ], [AC_MSG_ERROR([Cannot find proper lz4 version (>= 1.8.0)])])
+    LDFLAGS=${saved_LDFLAGS}
+
+    if test "x${have_lz4}" = "xyes"; then
+      AC_DEFINE([LZ4_ENABLED], [1], [Define to 1 if lz4 is enabled.])
+
+      if test "x${have_lz4hc}" = "xyes"; then
+        AC_DEFINE([LZ4HC_ENABLED], [1], [Define to 1 if lz4hc is enabled.])
+      fi
+
+      if test "x${lz4_force_static}" = "xyes"; then
+        LDFLAGS="-all-static ${LDFLAGS}"
+      else
+	test -z "${with_lz4_libdir}" || LZ4_LIBS="-R ${with_lz4_libdir} $LZ4_LIBS"
+      fi
+      LIBS="$LZ4_LIBS $LIBS"
+    fi
+  fi
+  CFLAGS=${saved_CPPFLAGS}
+fi
+
+AM_CONDITIONAL([ENABLE_LZ4], [test "x${have_lz4}" = "xyes"])
+AM_CONDITIONAL([ENABLE_LZ4HC], [test "x${have_lz4hc}" = "xyes"])
+
 AC_CONFIG_FILES([Makefile
 		 lib/Makefile
 		 mkfs/Makefile])
 AC_OUTPUT
+
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 64da708..17dc9e1 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -4,4 +4,11 @@
 noinst_LTLIBRARIES = liberofs.la
 liberofs_la_SOURCES = config.c io.c cache.c inode.c compressor.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
+if ENABLE_LZ4
+liberofs_la_CFLAGS += ${LZ4_CFLAGS}
+liberofs_la_SOURCES += compressor_lz4.c
+if ENABLE_LZ4HC
+liberofs_la_SOURCES += compressor_lz4hc.c
+endif
+endif
 
diff --git a/lib/compressor.c b/lib/compressor.c
index cc97cfb..a6138c6 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -39,6 +39,12 @@ int erofs_compressor_init(struct erofs_compress *c,
 			  char *alg_name)
 {
 	static struct erofs_compressor *compressors[] = {
+#if LZ4_ENABLED
+#if LZ4HC_ENABLED
+		&erofs_compressor_lz4hc,
+#endif
+		&erofs_compressor_lz4,
+#endif
 	};
 
 	int ret, i;
diff --git a/lib/compressor.h b/lib/compressor.h
index 8ad9d11..814ef2b 100644
--- a/lib/compressor.h
+++ b/lib/compressor.h
@@ -35,8 +35,14 @@ struct erofs_compress {
 	unsigned int destsize_alignsize;
 	unsigned int destsize_redzone_begin;
 	unsigned int destsize_redzone_end;
+
+	void *private;
 };
 
+/* list of compression algorithms */
+extern struct erofs_compressor erofs_compressor_lz4;
+extern struct erofs_compressor erofs_compressor_lz4hc;
+
 int erofs_compress_destsize(struct erofs_compress *c, int compression_level,
 			    void *src, unsigned int *srcsize,
 			    void *dst, unsigned int dstsize);
diff --git a/lib/compressor_lz4.c b/lib/compressor_lz4.c
new file mode 100644
index 0000000..eacd21c
--- /dev/null
+++ b/lib/compressor_lz4.c
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/compressor-lz4.c
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#include <lz4.h>
+#include "erofs/internal.h"
+#include "compressor.h"
+
+static int lz4_compress_destsize(struct erofs_compress *c,
+				 int compression_level,
+				 void *src, unsigned int *srcsize,
+				 void *dst, unsigned int dstsize)
+{
+	int srcSize = (int)*srcsize;
+	int rc = LZ4_compress_destSize(src, dst, &srcSize, (int)dstsize);
+
+	if (!rc)
+		return -EFAULT;
+	*srcsize = srcSize;
+	return 0;
+}
+
+static int compressor_lz4_exit(struct erofs_compress *c)
+{
+	return 0;
+}
+
+static int compressor_lz4_init(struct erofs_compress *c,
+				 char *alg_name)
+{
+	if (alg_name && strcmp(alg_name, "lz4"))
+		return -EINVAL;
+	c->alg = &erofs_compressor_lz4;
+	return 0;
+}
+
+struct erofs_compressor erofs_compressor_lz4 = {
+	.default_level = 0,
+	.best_level = 0,
+	.init = compressor_lz4_init,
+	.exit = compressor_lz4_exit,
+	.compress_destsize = lz4_compress_destsize,
+};
+
diff --git a/lib/compressor_lz4hc.c b/lib/compressor_lz4hc.c
new file mode 100644
index 0000000..3bbb754
--- /dev/null
+++ b/lib/compressor_lz4hc.c
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/compressor-lz4hc.c
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#define LZ4_HC_STATIC_LINKING_ONLY (1)
+#include <lz4hc.h>
+#include "erofs/internal.h"
+#include "compressor.h"
+
+static int lz4hc_compress_destsize(struct erofs_compress *c,
+				   int compression_level,
+				   void *src,
+				   unsigned int *srcsize,
+				   void *dst,
+				   unsigned int dstsize)
+{
+	int srcSize = (int)*srcsize;
+	int rc = LZ4_compress_HC_destSize(c->private, src, dst,
+					  &srcSize, (int)dstsize,
+					  compression_level);
+	if (!rc)
+		return -EFAULT;
+	*srcsize = srcSize;
+	return 0;
+}
+
+static int compressor_lz4hc_exit(struct erofs_compress *c)
+{
+	if (!c->private)
+		return -EINVAL;
+
+	LZ4_freeStreamHC(c->private);
+	return 0;
+}
+
+static int compressor_lz4hc_init(struct erofs_compress *c,
+				 char *alg_name)
+{
+	if (alg_name && strcmp(alg_name, "lz4hc"))
+		return -EINVAL;
+
+	c->alg = &erofs_compressor_lz4hc;
+
+	c->private = LZ4_createStreamHC();
+	if (!c->private)
+		return -ENOMEM;
+	return 0;
+}
+
+struct erofs_compressor erofs_compressor_lz4hc = {
+	.default_level = LZ4HC_CLEVEL_DEFAULT,
+	.best_level = LZ4HC_CLEVEL_MAX,
+	.init = compressor_lz4hc_init,
+	.exit = compressor_lz4hc_exit,
+	.compress_destsize = lz4hc_compress_destsize,
+};
+
-- 
2.17.1

^ permalink raw reply related	[flat|nested] 14+ messages in thread
* [PATCH 10/13] erofs-utils: introduce compression for regular files
  2019-05-31  0:50 [PATCH 00/13] erofs_utils: new mkfs framework Gao Xiang
                   ` (8 preceding siblings ...)
  2019-05-31  0:50 ` [PATCH 09/13] erofs-utils: introduce lz4/lz4hc compression algorithm Gao Xiang
@ 2019-05-31  0:50 ` Gao Xiang
  2019-05-31  0:50 ` [PATCH 11/13] erofs-utils: propagate compressed size to its callers Gao Xiang
                   ` (2 subsequent siblings)
  12 siblings, 0 replies; 14+ messages in thread
From: Gao Xiang @ 2019-05-31  0:50 UTC (permalink / raw)


From: Gao Xiang <gaoxiang25@huawei.com>

Yay! Let's try to transparent file compression
for all regular files if necessary.

Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 include/erofs/compress.h |  25 ++++
 include/erofs/config.h   |   2 +
 include/erofs/internal.h |   1 +
 lib/Makefile.am          |   2 +-
 lib/compress.c           | 303 +++++++++++++++++++++++++++++++++++++++
 lib/config.c             |   1 +
 lib/inode.c              |  18 ++-
 mkfs/main.c              |  21 +++
 8 files changed, 369 insertions(+), 4 deletions(-)
 create mode 100644 include/erofs/compress.h
 create mode 100644 lib/compress.c

diff --git a/include/erofs/compress.h b/include/erofs/compress.h
new file mode 100644
index 0000000..e0abb8f
--- /dev/null
+++ b/include/erofs/compress.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_utils/include/erofs/compress.h
+ *
+ * Copyright (C) 2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#ifndef __EROFS_COMPRESS_H
+#define __EROFS_COMPRESS_H
+
+#include "internal.h"
+
+/* workaround for an upstream lz4 compression issue, which can crash us */
+/* #define EROFS_CONFIG_COMPR_MAX_SZ        (1024 * 1024) */
+#define EROFS_CONFIG_COMPR_MAX_SZ           (900  * 1024)
+#define EROFS_CONFIG_COMPR_MIN_SZ           (32   * 1024)
+
+int erofs_write_compressed_file(struct erofs_inode *inode);
+
+int z_erofs_compress_init(void);
+int z_erofs_compress_exit(void);
+
+#endif
+
diff --git a/include/erofs/config.h b/include/erofs/config.h
index e31732b..177017b 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -19,6 +19,8 @@ struct erofs_configure {
 	/* related arguments for mkfs.erofs */
 	char *c_img_path;
 	char *c_src_path;
+	char *c_compr_alg_master;
+	int c_compr_level_master;
 };
 
 extern struct erofs_configure cfg;
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index dfa6173..78da589 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -94,6 +94,7 @@ struct erofs_inode {
 	struct erofs_buffer_head *bh_inline, *bh_data;
 
 	void *idata;
+	void *compressmeta;
 };
 
 #define IS_ROOT(x)	((x) == (x)->i_parent)
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 17dc9e1..dea82f7 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -2,7 +2,7 @@
 # Makefile.am
 
 noinst_LTLIBRARIES = liberofs.la
-liberofs_la_SOURCES = config.c io.c cache.c inode.c compressor.c
+liberofs_la_SOURCES = config.c io.c cache.c inode.c compress.c compressor.c
 liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
 if ENABLE_LZ4
 liberofs_la_CFLAGS += ${LZ4_CFLAGS}
diff --git a/lib/compress.c b/lib/compress.c
new file mode 100644
index 0000000..205a4f2
--- /dev/null
+++ b/lib/compress.c
@@ -0,0 +1,303 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs_utils/lib/compress.c
+ *
+ * Copyright (C) 2018-2019 HUAWEI, Inc.
+ *             http://www.huawei.com/
+ * Created by Miao Xie <miaoxie at huawei.com>
+ * with heavy changes by Gao Xiang <gaoxiang25 at huawei.com>
+ */
+#define _LARGEFILE64_SOURCE
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include "erofs/print.h"
+#include "erofs/io.h"
+#include "erofs/cache.h"
+#include "erofs/compress.h"
+#include "compressor.h"
+
+static struct erofs_compress compresshandle;
+static int compressionlevel;
+
+struct z_erofs_vle_compress_ctx {
+	u8 *metacur;
+
+	u8 queue[EROFS_CONFIG_COMPR_MAX_SZ * 2];
+	unsigned int head, tail;
+
+	erofs_blk_t blkaddr;	/* pointing to the next blkaddr */
+	u16 clusterofs;
+};
+
+static unsigned int get_vle_compress_metasize(erofs_off_t filesize)
+{
+	const unsigned int indexsize = BLK_ROUND_UP(filesize) *
+		sizeof(struct z_erofs_vle_decompressed_index);
+
+	return sizeof(struct erofs_extent_header) + indexsize;
+}
+
+static void vle_write_indexes_final(struct z_erofs_vle_compress_ctx *ctx)
+{
+	const unsigned int type = Z_EROFS_VLE_CLUSTER_TYPE_PLAIN;
+	struct z_erofs_vle_decompressed_index di;
+
+	if (!ctx->clusterofs)
+		return;
+
+	di.di_clusterofs = cpu_to_le16(ctx->clusterofs);
+	di.di_u.blkaddr = 0;
+	di.di_advise = cpu_to_le16(type << Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT);
+
+	memcpy(ctx->metacur, &di, sizeof(di));
+	ctx->metacur += sizeof(di);
+}
+
+static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx,
+			      unsigned int count, bool raw)
+{
+	unsigned int d0 = 0, d1 = (ctx->clusterofs + count) / EROFS_BLKSIZ;
+	unsigned int clusterofs = ctx->clusterofs;
+	struct z_erofs_vle_decompressed_index di;
+	unsigned int type;
+	__le16 advise;
+
+	di.di_clusterofs = cpu_to_le16(ctx->clusterofs);
+
+	/* whether the tail-end (un)compressed block or not */
+	if (clusterofs + count < EROFS_BLKSIZ) {
+		type = raw ? Z_EROFS_VLE_CLUSTER_TYPE_PLAIN :
+			Z_EROFS_VLE_CLUSTER_TYPE_HEAD;
+		advise = cpu_to_le16(type << Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT);
+
+		di.di_advise = advise;
+		di.di_u.blkaddr = cpu_to_le32(ctx->blkaddr);
+		memcpy(ctx->metacur, &di, sizeof(di));
+		ctx->metacur += sizeof(di);
+
+		/* don't add the final index if the tail-end block exists */
+		ctx->clusterofs = 0;
+		return;
+	}
+
+	do {
+		if (d0) {
+			type = Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD;
+
+			di.di_u.delta[0] = cpu_to_le16(d0);
+			di.di_u.delta[1] = cpu_to_le16(d1);
+		} else {
+			type = raw ? Z_EROFS_VLE_CLUSTER_TYPE_PLAIN :
+				Z_EROFS_VLE_CLUSTER_TYPE_HEAD;
+			di.di_u.blkaddr = cpu_to_le32(ctx->blkaddr);
+		}
+		advise = cpu_to_le16(type << Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT);
+		di.di_advise = advise;
+
+		memcpy(ctx->metacur, &di, sizeof(di));
+		ctx->metacur += sizeof(di);
+
+		count -= EROFS_BLKSIZ - clusterofs;
+		clusterofs = 0;
+
+		++d0;
+		--d1;
+	} while (clusterofs + count >= EROFS_BLKSIZ);
+
+	ctx->clusterofs = clusterofs + count;
+}
+
+static int vle_compress_one(struct erofs_inode *inode,
+			    struct z_erofs_vle_compress_ctx *ctx,
+			    bool final)
+{
+	struct erofs_compress *const h = &compresshandle;
+	unsigned int len = ctx->tail - ctx->head;
+	unsigned int count;
+	int ret;
+	char dst[EROFS_BLKSIZ];
+
+	while (len) {
+		bool raw;
+
+		if (len <= EROFS_BLKSIZ) {
+			if (final)
+				goto nocompression;
+			break;
+		}
+
+		count = len;
+		ret = erofs_compress_destsize(h, compressionlevel,
+					      ctx->queue + ctx->head,
+					      &count, dst, EROFS_BLKSIZ);
+		if (ret) {
+			if (ret != -EAGAIN) {
+				erofs_err("failed to compress %s: %s",
+					  inode->i_srcpath,
+					  erofs_strerror(ret));
+			}
+nocompression:
+			/* fix up clusterofs to 0 if possable */
+			if (ctx->head >= ctx->clusterofs) {
+				ctx->head -= ctx->clusterofs;
+				len += ctx->clusterofs;
+				ctx->clusterofs = 0;
+			}
+
+			/* write uncompressed data */
+			count = min(EROFS_BLKSIZ, len);
+
+			memcpy(dst, ctx->queue + ctx->head, count);
+			memset(dst + count, 0, EROFS_BLKSIZ - count);
+
+			erofs_dbg("Writing %u uncompressed data to block %u",
+				  count, ctx->blkaddr);
+
+			ret = blk_write(dst, ctx->blkaddr, 1);
+			if (ret)
+				return ret;
+			raw = true;
+		} else {
+			/* write compressed data */
+			erofs_dbg("Writing %u compressed data to block %u",
+				  count, ctx->blkaddr);
+
+			ret = blk_write(dst, ctx->blkaddr, 1);
+			if (ret)
+				return ret;
+			raw = false;
+		}
+
+		ctx->head += count;
+		/* write compression indexes for this blkaddr */
+		vle_write_indexes(ctx, count, raw);
+
+		++ctx->blkaddr;
+		len -= count;
+
+		if (!final && ctx->head >= EROFS_CONFIG_COMPR_MAX_SZ) {
+			const uint qh_aligned = round_down(ctx->head, EROFS_BLKSIZ);
+			const uint qh_after = ctx->head - qh_aligned;
+
+			memmove(ctx->queue, ctx->queue + qh_aligned,
+				len + qh_after);
+			ctx->head = qh_after;
+			ctx->tail = qh_after + len;
+			break;
+		}
+	}
+	return 0;
+}
+
+int erofs_write_compressed_file(struct erofs_inode *inode)
+{
+	const unsigned int metasize = get_vle_compress_metasize(inode->i_size);
+	struct erofs_buffer_head *bh;
+	struct z_erofs_vle_compress_ctx ctx;
+	erofs_off_t remaining;
+	erofs_blk_t blkaddr;
+
+	int ret, fd;
+	u8 *compressmeta = malloc(metasize);
+
+	if (!compressmeta)
+		return -ENOMEM;
+
+	fd = open(inode->i_srcpath, O_RDONLY | O_BINARY);
+	if (fd < 0) {
+		ret = -errno;
+		goto err_free;
+	}
+
+	/* allocate main data buffer */
+	bh = erofs_balloc(DATA, 0, 0, 0);
+	if (IS_ERR(bh)) {
+		ret = PTR_ERR(bh);
+		goto err_close;
+	}
+
+	blkaddr = erofs_mapbh(bh->block, true);	/* start_blkaddr */
+	ctx.blkaddr = blkaddr;
+	ctx.metacur = compressmeta + sizeof(struct erofs_extent_header);
+	ctx.head = ctx.tail = 0;
+	ctx.clusterofs = 0;
+	remaining = inode->i_size;
+
+	while (remaining) {
+		const uint readcount = min_t(uint, remaining,
+					     sizeof(ctx.queue) - ctx.tail);
+
+		ret = read(fd, ctx.queue + ctx.tail, readcount);
+		if (ret != readcount) {
+			ret = -errno;
+			goto err_bdrop;
+		}
+		remaining -= readcount;
+		ctx.tail += readcount;
+
+		/* do one compress round */
+		ret = vle_compress_one(inode, &ctx, false);
+		if (ret)
+			goto err_bdrop;
+	}
+
+	/* do the final round */
+	ret = vle_compress_one(inode, &ctx, true);
+	if (ret)
+		goto err_bdrop;
+
+	/* fall back to no compression mode */
+	if (ctx.blkaddr - blkaddr >= BLK_ROUND_UP(inode->i_size)) {
+		ret = -ENOSPC;
+		goto err_bdrop;
+	}
+
+	vle_write_indexes_final(&ctx);
+
+	close(fd);
+	ret = erofs_bh_balloon(bh, blknr_to_addr(ctx.blkaddr - blkaddr));
+	DBG_BUGON(ret);
+
+	erofs_info("compressed %s (%lu bytes) into %u blocks",
+		   inode->i_srcpath, inode->i_size, ctx.blkaddr - blkaddr);
+
+	/*
+	 * TODO: need to move erofs_bdrop to erofs_write_tail_end
+	 *       when both mkfs & kernel support compression inline.
+	 */
+	erofs_bdrop(bh, false);
+	inode->compressmeta = compressmeta;
+	inode->idata_size = 0;
+	inode->extent_isize = metasize;
+	inode->data_mapping_mode = EROFS_INODE_LAYOUT_COMPRESSION;
+	return 0;
+
+err_bdrop:
+	erofs_bdrop(bh, true);	/* revoke buffer */
+err_close:
+	close(fd);
+err_free:
+	free(compressmeta);
+	return ret;
+}
+
+int z_erofs_compress_init(void)
+{
+	/* initialize for primary compression algorithm */
+	int ret = erofs_compressor_init(&compresshandle,
+					cfg.c_compr_alg_master);
+	if (ret)
+		return ret;
+
+	compressionlevel = cfg.c_compr_level_master < 0 ?
+		compresshandle.alg->default_level :
+		cfg.c_compr_level_master;
+	return 0;
+}
+
+int z_erofs_compress_exit(void)
+{
+	return erofs_compressor_exit(&compresshandle);
+}
+
diff --git a/lib/config.c b/lib/config.c
index 6ff8e4d..3312c9b 100644
--- a/lib/config.c
+++ b/lib/config.c
@@ -18,6 +18,7 @@ void erofs_init_configure(void)
 	cfg.c_dbg_lvl  = 0;
 	cfg.c_version  = PACKAGE_VERSION;
 	cfg.c_dry_run  = false;
+	cfg.c_compr_level_master = -1;
 }
 
 void erofs_show_config(void)
diff --git a/lib/inode.c b/lib/inode.c
index d3e5ed8..b7fcf0e 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -17,6 +17,7 @@
 #include "erofs/inode.h"
 #include "erofs/cache.h"
 #include "erofs/io.h"
+#include "erofs/compress.h"
 
 struct erofs_sb_info sbi;
 
@@ -286,8 +287,11 @@ int erofs_write_file(struct erofs_inode *inode)
 	unsigned int nblocks, i;
 	int ret, fd;
 
-	if (erofs_file_is_compressible(inode)) {
-		/* to be implemented */
+	if (cfg.c_compr_alg_master && erofs_file_is_compressible(inode)) {
+		ret = erofs_write_compressed_file(inode);
+
+		if (!ret || ret != -ENOSPC)
+			return ret;
 	}
 
 	/* fallback to all data uncompressed */
@@ -340,7 +344,7 @@ fail:
 static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)
 {
 	struct erofs_inode *const inode = bh->fsprivate;
-	const erofs_off_t off = erofs_btell(bh, false);
+	erofs_off_t off = erofs_btell(bh, false);
 
 	/* let's support v1 currently */
 	struct erofs_inode_v1 v1 = {0};
@@ -379,6 +383,14 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)
 	if (ret)
 		return false;
 
+	if (inode->extent_isize) {
+		/* write compression metadata */
+		off = Z_EROFS_VLE_EXTENT_ALIGN(off + inode->inode_isize);
+		ret = dev_write(inode->compressmeta, off, inode->extent_isize);
+		if (ret)
+			return false;
+	}
+
 	erofs_iput(inode);
 	return erofs_bh_flush_generic_end(bh);
 }
diff --git a/mkfs/main.c b/mkfs/main.c
index 1ed15d2..ce64ece 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -17,6 +17,7 @@
 #include "erofs/cache.h"
 #include "erofs/inode.h"
 #include "erofs/io.h"
+#include "erofs/compress.h"
 
 #define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
 
@@ -25,6 +26,7 @@ static void usage(char *execpath)
 	fprintf(stderr, "%s %s\n", basename(execpath), cfg.c_version);
 	fprintf(stderr, "usage: [options] FILE DIRECTORY\n\n");
 	fprintf(stderr, "Generate erofs image from DIRECTORY to FILE, and [options] are:\n");
+	fprintf(stderr, " -zX[,Y]  X=compressor (Y=compression level, optional)\n");
 	fprintf(stderr, " -d#      set output message level to # (maximum 9)\n");
 }
 
@@ -44,6 +46,23 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
 
 	while ((opt = getopt(argc, argv, "d:z:")) != -1) {
 		switch (opt) {
+		case 'z':
+			if (!optarg) {
+				cfg.c_compr_alg_master = "(default)";
+				break;
+			}
+			/* get specified compression level */
+			for (i = 0; optarg[i] != '\0'; ++i) {
+				if (optarg[i] == ',') {
+					cfg.c_compr_level_master =
+						atoi(optarg + i + 1);
+					optarg[i] = '\0';
+					break;
+				}
+			}
+			cfg.c_compr_alg_master = strndup(optarg, i);
+			break;
+
 		case 'd':
 			cfg.c_dbg_lvl = parse_num_from_str(optarg);
 			break;
@@ -148,6 +167,7 @@ int main(int argc, char **argv)
 		goto exit;
 	}
 
+	z_erofs_compress_init();
 	erofs_inode_manager_init();
 
 	root_inode = erofs_mkfs_build_tree_from_path(NULL, cfg.c_src_path);
@@ -167,6 +187,7 @@ int main(int argc, char **argv)
 	if (!erofs_bflush(NULL))
 		err = -EIO;
 exit:
+	z_erofs_compress_exit();
 	dev_close();
 	erofs_exit_configure();
 
-- 
2.17.1

^ permalink raw reply related	[flat|nested] 14+ messages in thread
* [PATCH 11/13] erofs-utils: propagate compressed size to its callers
  2019-05-31  0:50 [PATCH 00/13] erofs_utils: new mkfs framework Gao Xiang
                   ` (9 preceding siblings ...)
  2019-05-31  0:50 ` [PATCH 10/13] erofs-utils: introduce compression for regular files Gao Xiang
@ 2019-05-31  0:50 ` Gao Xiang
  2019-05-31  0:50 ` [PATCH 12/13] erofs-utils: add a README Gao Xiang
  2019-05-31  0:50 ` [PATCH 13/13] erofs-utils: fix potential erofs_bh_balloon failure Gao Xiang
  12 siblings, 0 replies; 14+ messages in thread
From: Gao Xiang @ 2019-05-31  0:50 UTC (permalink / raw)


From: Gao Xiang <gaoxiang25@huawei.com>

It will be later used for marking whether it can
decompress in-place.

Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 lib/compress.c         | 2 +-
 lib/compressor.c       | 4 ++--
 lib/compressor_lz4.c   | 2 +-
 lib/compressor_lz4hc.c | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/compress.c b/lib/compress.c
index 205a4f2..7bfb8ce 100644
--- a/lib/compress.c
+++ b/lib/compress.c
@@ -131,7 +131,7 @@ static int vle_compress_one(struct erofs_inode *inode,
 		ret = erofs_compress_destsize(h, compressionlevel,
 					      ctx->queue + ctx->head,
 					      &count, dst, EROFS_BLKSIZ);
-		if (ret) {
+		if (ret <= 0) {
 			if (ret != -EAGAIN) {
 				erofs_err("failed to compress %s: %s",
 					  inode->i_srcpath,
diff --git a/lib/compressor.c b/lib/compressor.c
index a6138c6..570db14 100644
--- a/lib/compressor.c
+++ b/lib/compressor.c
@@ -26,13 +26,13 @@ int erofs_compress_destsize(struct erofs_compress *c,
 
 	ret = c->alg->compress_destsize(c, compression_level,
 					src, srcsize, dst, dstsize);
-	if (ret)
+	if (ret < 0)
 		return ret;
 
 	/* check if there is enough gains to compress */
 	if (*srcsize <= dstsize * c->compress_threshold / 100)
 		return -EAGAIN;
-	return 0;
+	return ret;
 }
 
 int erofs_compressor_init(struct erofs_compress *c,
diff --git a/lib/compressor_lz4.c b/lib/compressor_lz4.c
index eacd21c..0d33223 100644
--- a/lib/compressor_lz4.c
+++ b/lib/compressor_lz4.c
@@ -21,7 +21,7 @@ static int lz4_compress_destsize(struct erofs_compress *c,
 	if (!rc)
 		return -EFAULT;
 	*srcsize = srcSize;
-	return 0;
+	return rc;
 }
 
 static int compressor_lz4_exit(struct erofs_compress *c)
diff --git a/lib/compressor_lz4hc.c b/lib/compressor_lz4hc.c
index 3bbb754..d9413e2 100644
--- a/lib/compressor_lz4hc.c
+++ b/lib/compressor_lz4hc.c
@@ -25,7 +25,7 @@ static int lz4hc_compress_destsize(struct erofs_compress *c,
 	if (!rc)
 		return -EFAULT;
 	*srcsize = srcSize;
-	return 0;
+	return rc;
 }
 
 static int compressor_lz4hc_exit(struct erofs_compress *c)
-- 
2.17.1

^ permalink raw reply related	[flat|nested] 14+ messages in thread
* [PATCH 12/13] erofs-utils: add a README
  2019-05-31  0:50 [PATCH 00/13] erofs_utils: new mkfs framework Gao Xiang
                   ` (10 preceding siblings ...)
  2019-05-31  0:50 ` [PATCH 11/13] erofs-utils: propagate compressed size to its callers Gao Xiang
@ 2019-05-31  0:50 ` Gao Xiang
  2019-05-31  0:50 ` [PATCH 13/13] erofs-utils: fix potential erofs_bh_balloon failure Gao Xiang
  12 siblings, 0 replies; 14+ messages in thread
From: Gao Xiang @ 2019-05-31  0:50 UTC (permalink / raw)


From: Gao Xiang <gaoxiang25@huawei.com>

This patch adds a README and some warnings here.

Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
 README | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 README

diff --git a/README b/README
new file mode 100644
index 0000000..4d509be
--- /dev/null
+++ b/README
@@ -0,0 +1,11 @@
+
+The new erofs-utils introduces a completely new framework,
+which is more cleaner but still under development for now.
+
+Use the old the mkfs-dev branch of erofs-utils temporarily
+to build erofs image instead:
+git clone git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git -b mkfs-dev
+
+Thanks,
+Gao Xiang
+
-- 
2.17.1

^ permalink raw reply related	[flat|nested] 14+ messages in thread
* [PATCH 13/13] erofs-utils: fix potential erofs_bh_balloon failure
  2019-05-31  0:50 [PATCH 00/13] erofs_utils: new mkfs framework Gao Xiang
                   ` (11 preceding siblings ...)
  2019-05-31  0:50 ` [PATCH 12/13] erofs-utils: add a README Gao Xiang
@ 2019-05-31  0:50 ` Gao Xiang
  12 siblings, 0 replies; 14+ messages in thread
From: Gao Xiang @ 2019-05-31  0:50 UTC (permalink / raw)


From: Gao Xiang <gaoxiang25@huawei.com>

erofs_write_tail_end is called at the last end
of erofs_mkfs_build_tree, which is of course
after traversing all sub-tree inodes.

Therefore, erofs_bh_balloon could fail if
corresponding block is allocated, fix it.

Signed-off-by: Gao Xiang <gaoxiang25 at huawei.com>
---
Will be wrapped in the original patch.

 lib/inode.c | 61 +++++++++++++++++++++++++++++++++++------------------
 1 file changed, 40 insertions(+), 21 deletions(-)

diff --git a/lib/inode.c b/lib/inode.c
index b7fcf0e..9f7a4e4 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -399,6 +399,35 @@ static struct erofs_bhops erofs_write_inode_bhops = {
 	.flush = erofs_bh_flush_write_inode,
 };
 
+int erofs_prepare_tail_block(struct erofs_inode *inode)
+{
+	struct erofs_buffer_head *bh;
+	int ret;
+
+	if (!inode->idata_size)
+		return 0;
+
+	bh = inode->bh_data;
+	if (!bh) {
+		bh = erofs_balloc(DATA, EROFS_BLKSIZ, 0, 0);
+		if (IS_ERR(bh))
+			return PTR_ERR(bh);
+		bh->op = &erofs_skip_write_bhops;
+
+		/* get blkaddr of bh */
+		ret = erofs_mapbh(bh->block, true);
+		DBG_BUGON(ret < 0);
+		inode->u.i_blkaddr = bh->block->blkaddr;
+
+		inode->bh_data = bh;
+		return 0;
+	}
+	/* expend a block as the tail block (should be successful) */
+	ret = erofs_bh_balloon(bh, EROFS_BLKSIZ);
+	DBG_BUGON(ret);
+	return 0;
+}
+
 int erofs_prepare_inode_buffer(struct erofs_inode *inode)
 {
 	unsigned int inodesize;
@@ -421,11 +450,18 @@ int erofs_prepare_inode_buffer(struct erofs_inode *inode)
 
 	bh = erofs_balloc(INODE, inodesize, 0, inode->idata_size);
 	if (bh == ERR_PTR(-ENOSPC)) {
+		int ret;
+
 		inode->data_mapping_mode = EROFS_INODE_LAYOUT_PLAIN;
 noinline:
+		/* expend an extra block for tail-end data */
+		ret = erofs_prepare_tail_block(inode);
+		if (ret)
+			return ret;
 		bh = erofs_balloc(INODE, inodesize, 0, 0);
 		if (IS_ERR(bh))
 			return PTR_ERR(bh);
+		DBG_BUGON(inode->bh_inline);
 	} else if (IS_ERR(bh)) {
 		return PTR_ERR(bh);
 	} else if (inode->idata_size) {
@@ -485,28 +521,11 @@ int erofs_write_tail_end(struct erofs_inode *inode)
 		ibh->op = &erofs_write_inline_bhops;
 	} else {
 		int ret;
-		erofs_off_t off;
-
-		if (bh) {
-			/* expend a block (should be successful) */
-			ret = erofs_bh_balloon(bh, EROFS_BLKSIZ);
-			DBG_BUGON(ret);
-
-			erofs_mapbh(bh->block, true);
-			off = erofs_btell(bh, true) - EROFS_BLKSIZ;
-		} else {
-			bh = erofs_balloc(DATA, EROFS_BLKSIZ, 0, 0);
-			if (IS_ERR(bh))
-				return PTR_ERR(bh);
-
-			bh->op = &erofs_skip_write_bhops;
-			erofs_mapbh(bh->block, true);
-			off = erofs_btell(bh, false);
-			inode->u.i_blkaddr = erofs_blknr(off);
-			inode->bh_data = bh;
-		}
 
-		ret = dev_write(inode->idata, off, inode->idata_size);
+		erofs_mapbh(bh->block, true);
+		ret = dev_write(inode->idata,
+				erofs_btell(bh, true) - EROFS_BLKSIZ,
+				inode->idata_size);
 		if (ret)
 			return ret;
 
-- 
2.17.1

^ permalink raw reply related	[flat|nested] 14+ messages in thread
end of thread, other threads:[~2019-05-31  0:50 UTC | newest]

Thread overview: 14+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2019-05-31  0:50 [PATCH 00/13] erofs_utils: new mkfs framework Gao Xiang
2019-05-31  0:50 ` [PATCH 01/13] erofs-utils: add erofs on-disk layout Gao Xiang
2019-05-31  0:50 ` [PATCH 02/13] erofs-utils: introduce erofs-utils basic headers Gao Xiang
2019-05-31  0:50 ` [PATCH 03/13] erofs-utils: introduce miscellaneous files Gao Xiang
2019-05-31  0:50 ` [PATCH 04/13] erofs-utils: add input/output functions Gao Xiang
2019-05-31  0:50 ` [PATCH 05/13] erofs-utils: introduce buffer cache Gao Xiang
2019-05-31  0:50 ` [PATCH 06/13] erofs-utils: introduce inode operations Gao Xiang
2019-05-31  0:50 ` [PATCH 07/13] erofs-utils: introduce mkfs support Gao Xiang
2019-05-31  0:50 ` [PATCH 08/13] erofs-utils: introduce generic compression framework Gao Xiang
2019-05-31  0:50 ` [PATCH 09/13] erofs-utils: introduce lz4/lz4hc compression algorithm Gao Xiang
2019-05-31  0:50 ` [PATCH 10/13] erofs-utils: introduce compression for regular files Gao Xiang
2019-05-31  0:50 ` [PATCH 11/13] erofs-utils: propagate compressed size to its callers Gao Xiang
2019-05-31  0:50 ` [PATCH 12/13] erofs-utils: add a README Gao Xiang
2019-05-31  0:50 ` [PATCH 13/13] erofs-utils: fix potential erofs_bh_balloon failure Gao Xiang

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.