* [PREVIEW] [RFC PATCH v2 0/5] erofs-mkfs: mkfs.erofs source code first preview
@ 2018-11-20 14:32 Li Guifu
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 1/5] erofs-mkfs: add erofs on-disk layout Li Guifu
` (4 more replies)
0 siblings, 5 replies; 6+ messages in thread
From: Li Guifu @ 2018-11-20 14:32 UTC (permalink / raw)
This patchset introduces the source code for mkfs.erofs. It can build
erofs image from scratch according to a specific directory.
Currently, it can build lz4 compressed images and uncompressed images.
Regular files, directories and special files are supported.
However it can only generate erofs inode v1 format due to our product
requirements, and inode v2 format support is still under development.
This is our first attempt to release this piece of code, which isn't
clean enough. It will be cleaned up with our community development.
Comments are welcome.
Thanks,
Li Guifu
change log from v1:
- fix coding style reported by checkpatch.pl
- fix image size overflow when it's bigger than 4GB
Li Guifu (5):
erofs-mkfs: add erofs on-disk layout
erofs-mkfs: introduce mkfs basic framework
erofs-mkfs: support to build a image from the specific root directory
erofs-mkfs: add compression support
erofs-mkfs: add miscellaneous files
--
2.17.1
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PREVIEW] [RFC PATCH v2 1/5] erofs-mkfs: add erofs on-disk layout
2018-11-20 14:32 [PREVIEW] [RFC PATCH v2 0/5] erofs-mkfs: mkfs.erofs source code first preview Li Guifu
@ 2018-11-20 14:32 ` Li Guifu
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 2/5] erofs-mkfs: introduce mkfs basic framework Li Guifu
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Li Guifu @ 2018-11-20 14:32 UTC (permalink / raw)
This patch adds a header file describing the erofs on-disk layout, which
should be kept in line with the kernel implementation all the time.
Signed-off-by: Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Fang Wei <fangwei1 at huawei.com>
---
erofs_fs.h | 276 +++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 276 insertions(+)
create mode 100644 erofs_fs.h
diff --git a/erofs_fs.h b/erofs_fs.h
new file mode 100644
index 0000000..e68f206
--- /dev/null
+++ b/erofs_fs.h
@@ -0,0 +1,276 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_fs.h
+ *
+ * Copyright (C) 2017-2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_FS_H
+#define __EROFS_FS_H
+
+
+
+
+/* Enhanced(Extended) ROM File System */
+#define EROFS_SUPER_MAGIC_V1 0xE0F5E1E2
+#define EROFS_SUPER_OFFSET 1024
+
+struct erofs_super_block {
+/* 0 */__le32 magic; /* in the little endian */
+/* 4 */__le32 checksum; /* crc32c(super_block) */
+/* 8 */__le32 features;
+/* 12 */__u8 blkszbits; /* support block_size == PAGE_SIZE only */
+/* 13 */__u8 reserved;
+
+/* 14 */__le16 root_nid;
+/* 16 */__le64 inos; /* total valid ino # (== f_files - f_favail) */
+
+/* 24 */__le64 build_time; /* inode v1 time derivation */
+/* 32 */__le32 build_time_nsec;
+/* 36 */__le32 blocks; /* used for statfs */
+/* 40 */__le32 meta_blkaddr;
+/* 44 */__le32 xattr_blkaddr;
+/* 48 */__u8 uuid[16]; /* 128-bit uuid for volume */
+/* 64 */__u8 volume_name[16]; /* volume name */
+
+/* 80 */__u8 reserved2[48]; /* 128 bytes */
+} __packed;
+
+#define __EROFS_BIT(_prefix, _cur, _pre) enum { \
+ _prefix ## _cur ## _BIT = _prefix ## _pre ## _BIT + \
+ _prefix ## _pre ## _BITS }
+
+/*
+ * erofs inode data mapping:
+ * 0 - inode plain without inline data A:
+ * inode, [xattrs], ... | ... | no-holed data
+ * 1 - inode VLE compression B:
+ * inode, [xattrs], extents ... | ...
+ * 2 - inode plain with inline data C:
+ * inode, [xattrs], last_inline_data, ... | ... | no-holed data
+ * 3~7 - reserved
+ */
+enum {
+ EROFS_INODE_LAYOUT_PLAIN,
+ EROFS_INODE_LAYOUT_COMPRESSION,
+ EROFS_INODE_LAYOUT_INLINE,
+ EROFS_INODE_LAYOUT_MAX
+};
+#define EROFS_I_VERSION_BITS 1
+#define EROFS_I_DATA_MAPPING_BITS 3
+
+#define EROFS_I_VERSION_BIT 0
+__EROFS_BIT(EROFS_I_, DATA_MAPPING, VERSION);
+
+struct erofs_inode_v1 {
+/* 0 */__le16 i_advise;
+
+/* 1 header + n-1 * 4 bytes inline xattr to keep continuity */
+/* 2 */__le16 i_xattr_icount;
+/* 4 */__le16 i_mode;
+/* 6 */__le16 i_nlink;
+/* 8 */__le32 i_size;
+/* 12 */__le32 i_reserved;
+/* 16 */union {
+ /* file total compressed blocks for data mapping 1 */
+ __le32 compressed_blocks;
+ __le32 raw_blkaddr;
+
+ /* for device files, used to indicate old/new device # */
+ __le32 rdev;
+ } i_u __packed;
+/* 20 */__le32 i_ino; /* only used for 32-bit stat compatibility */
+/* 24 */__le16 i_uid;
+/* 26 */__le16 i_gid;
+/* 28 */__le32 i_checksum;
+} __packed;
+
+/* 32 bytes on-disk inode */
+#define EROFS_INODE_LAYOUT_V1 0
+/* 64 bytes on-disk inode */
+#define EROFS_INODE_LAYOUT_V2 1
+
+struct erofs_inode_v2 {
+ __le16 i_advise;
+
+ /* 1 header + n-1 * 4 bytes inline xattr to keep continuity */
+ __le16 i_xattr_icount;
+ __le16 i_mode;
+ __le16 i_reserved; /* 8 bytes */
+ __le64 i_size; /* 16 bytes */
+ union {
+ /* file total compressed blocks for data mapping 1 */
+ __le32 compressed_blocks;
+ __le32 raw_blkaddr;
+
+ /* for device files, used to indicate old/new device # */
+ __le32 rdev;
+ } i_u __packed;
+
+ /* only used for 32-bit stat compatibility */
+ __le32 i_ino; /* 24 bytes */
+
+ __le32 i_uid;
+ __le32 i_gid;
+ __le64 i_ctime; /* 32 bytes */
+ __le32 i_ctime_nsec;
+ __le32 i_nlink;
+ __u8 i_reserved2[12];
+ __le32 i_checksum; /* 64 bytes */
+} __packed;
+
+#define EROFS_MAX_SHARED_XATTRS (128)
+/* h_shared_count between 129 ... 255 are special # */
+#define EROFS_SHARED_XATTR_EXTENT (255)
+
+/*
+ * inline xattrs (n == i_xattr_icount):
+ * erofs_xattr_ibody_header(1) + (n - 1) * 4 bytes
+ * 12 bytes / \
+ * / \
+ * /-----------------------\
+ * | erofs_xattr_entries+ |
+ * +-----------------------+
+ * inline xattrs must starts in erofs_xattr_ibody_header,
+ * for read-only fs, no need to introduce h_refcount
+ */
+struct erofs_xattr_ibody_header {
+ __le32 h_checksum;
+ __u8 h_shared_count;
+ __u8 h_reserved[7];
+ __le32 h_shared_xattrs[0]; /* shared xattr id array */
+} __packed;
+
+/* Name indexes */
+#define EROFS_XATTR_INDEX_USER 1
+#define EROFS_XATTR_INDEX_POSIX_ACL_ACCESS 2
+#define EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT 3
+#define EROFS_XATTR_INDEX_TRUSTED 4
+#define EROFS_XATTR_INDEX_LUSTRE 5
+#define EROFS_XATTR_INDEX_SECURITY 6
+
+/* xattr entry (for both inline & shared xattrs) */
+struct erofs_xattr_entry {
+ __u8 e_name_len; /* length of name */
+ __u8 e_name_index; /* attribute name index */
+ __le16 e_value_size; /* size of attribute value */
+ /* followed by e_name and e_value */
+ char e_name[0]; /* attribute name */
+} __packed;
+
+#define ondisk_xattr_ibody_size(count) ({\
+ u32 __count = le16_to_cpu(count); \
+ ((__count) == 0) ? 0 : \
+ sizeof(struct erofs_xattr_ibody_header) + \
+ sizeof(__u32) * ((__count) - 1); })
+
+#define EROFS_XATTR_ALIGN(size) round_up(size, sizeof(struct erofs_xattr_entry))
+#define EROFS_XATTR_ENTRY_SIZE(entry) EROFS_XATTR_ALIGN( \
+ sizeof(struct erofs_xattr_entry) + \
+ (entry)->e_name_len + le16_to_cpu((entry)->e_value_size))
+
+/* have to be aligned with 8 bytes on disk */
+struct erofs_extent_header {
+ __le32 eh_checksum;
+ __le32 eh_reserved[3];
+} __packed;
+
+/*
+ * Z_EROFS Variable-sized Logical Extent cluster type:
+ * 0 - literal (uncompressed) cluster
+ * 1 - compressed cluster (for the head logical cluster)
+ * 2 - compressed cluster (for the other logical clusters)
+ *
+ * In detail,
+ * 0 - literal (uncompressed) cluster,
+ * di_advise = 0
+ * di_clusterofs = the literal data offset of the cluster
+ * di_blkaddr = the blkaddr of the literal cluster
+ *
+ * 1 - compressed cluster (for the head logical cluster)
+ * di_advise = 1
+ * di_clusterofs = the decompressed data offset of the cluster
+ * di_blkaddr = the blkaddr of the compressed cluster
+ *
+ * 2 - compressed cluster (for the other logical clusters)
+ * di_advise = 2
+ * di_clusterofs =
+ * the decompressed data offset in its own head cluster
+ * di_u.delta[0] = distance to its corresponding head cluster
+ * di_u.delta[1] = distance to its corresponding tail cluster
+ * (di_advise could be 0, 1 or 2)
+ */
+enum {
+ Z_EROFS_VLE_CLUSTER_TYPE_PLAIN,
+ Z_EROFS_VLE_CLUSTER_TYPE_HEAD,
+ Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD,
+ Z_EROFS_VLE_CLUSTER_TYPE_RESERVED,
+ Z_EROFS_VLE_CLUSTER_TYPE_MAX
+};
+
+#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS 2
+#define Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT 0
+
+struct z_erofs_vle_decompressed_index {
+ __le16 di_advise;
+ /* where to decompress in the head cluster */
+ __le16 di_clusterofs;
+
+ union {
+ /* for the head cluster */
+ __le32 blkaddr;
+ /*
+ * for the rest clusters
+ * eg. for 4k page-sized cluster, maximum 4K*64k = 256M)
+ * [0] - pointing to the head cluster
+ * [1] - pointing to the tail cluster
+ */
+ __le16 delta[2];
+ } di_u __packed; /* 8 bytes */
+} __packed;
+
+#define Z_EROFS_VLE_EXTENT_ALIGN(size) round_up(size, \
+ sizeof(struct z_erofs_vle_decompressed_index))
+
+/* dirent sorts in alphabet order, thus we can do binary search */
+struct erofs_dirent {
+ __le64 nid; /* 0, node number */
+ __le16 nameoff; /* 8, start offset of file name */
+ __u8 file_type; /* 10, file type */
+ __u8 reserved; /* 11, reserved */
+} __packed;
+
+/* file types used in inode_info->flags */
+enum {
+ EROFS_FT_UNKNOWN,
+ EROFS_FT_REG_FILE,
+ EROFS_FT_DIR,
+ EROFS_FT_CHRDEV,
+ EROFS_FT_BLKDEV,
+ EROFS_FT_FIFO,
+ EROFS_FT_SOCK,
+ EROFS_FT_SYMLINK,
+ EROFS_FT_MAX
+};
+
+#define EROFS_NAME_LEN 255
+
+/* check the EROFS on-disk layout strictly at compile time */
+static inline void erofs_check_ondisk_layout_definitions(void)
+{
+ BUILD_BUG_ON(sizeof(struct erofs_super_block) != 128);
+ BUILD_BUG_ON(sizeof(struct erofs_inode_v1) != 32);
+ BUILD_BUG_ON(sizeof(struct erofs_inode_v2) != 64);
+ BUILD_BUG_ON(sizeof(struct erofs_xattr_ibody_header) != 12);
+ BUILD_BUG_ON(sizeof(struct erofs_xattr_entry) != 4);
+ BUILD_BUG_ON(sizeof(struct erofs_extent_header) != 16);
+ BUILD_BUG_ON(sizeof(struct z_erofs_vle_decompressed_index) != 8);
+ BUILD_BUG_ON(sizeof(struct erofs_dirent) != 12);
+
+ BUILD_BUG_ON(BIT(Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) <
+ Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1);
+}
+
+#endif
+
--
2.17.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [PREVIEW] [RFC PATCH v2 2/5] erofs-mkfs: introduce mkfs basic framework
2018-11-20 14:32 [PREVIEW] [RFC PATCH v2 0/5] erofs-mkfs: mkfs.erofs source code first preview Li Guifu
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 1/5] erofs-mkfs: add erofs on-disk layout Li Guifu
@ 2018-11-20 14:32 ` Li Guifu
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 3/5] erofs-mkfs: support to build a image from the specific root directory Li Guifu
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Li Guifu @ 2018-11-20 14:32 UTC (permalink / raw)
This patch adds basic code to generate erofs file systems and
empty images can be created properly.
Some parameters are also introduced in this patch, used for
setting the debugging level, whether to compress or not.
Signed-off-by: Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Fang Wei <fangwei1 at huawei.com>
---
erofs_config.c | 56 ++++++++++++++++
erofs_config.h | 44 +++++++++++++
erofs_debug.h | 67 +++++++++++++++++++
erofs_error.h | 25 +++++++
erofs_io.c | 173 +++++++++++++++++++++++++++++++++++++++++++++++++
erofs_io.h | 24 +++++++
erofs_types.h | 52 +++++++++++++++
list_head.h | 110 +++++++++++++++++++++++++++++++
mkfs_erofs.h | 66 +++++++++++++++++++
mkfs_main.c | 165 ++++++++++++++++++++++++++++++++++++++++++++++
10 files changed, 782 insertions(+)
create mode 100644 erofs_config.c
create mode 100644 erofs_config.h
create mode 100644 erofs_debug.h
create mode 100644 erofs_error.h
create mode 100644 erofs_io.c
create mode 100644 erofs_io.h
create mode 100644 erofs_types.h
create mode 100644 list_head.h
create mode 100644 mkfs_erofs.h
create mode 100644 mkfs_main.c
diff --git a/erofs_config.c b/erofs_config.c
new file mode 100644
index 0000000..952799c
--- /dev/null
+++ b/erofs_config.c
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs_config.c
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "erofs_config.h"
+
+struct erofs_configure erofs_cfg;
+
+
+void mkfs_init_configure(void)
+{
+ memset(&erofs_cfg, 0, sizeof(erofs_cfg));
+ erofs_cfg.c_alg_name = "none";
+ erofs_cfg.c_dbg_lvl = 0;
+ erofs_cfg.c_version = EROFS_MKFS_VERSION " " __DATE__ " " __TIME__;
+}
+
+void mkfs_dump_config(void)
+{
+ fprintf(stderr, "\tc_version: [%8s]\n",
+ erofs_cfg.c_version);
+ fprintf(stderr, "\tc_img_path: [%8s]\n",
+ erofs_cfg.c_img_path);
+ fprintf(stderr, "\tc_src_path: [%8s]\n",
+ erofs_cfg.c_src_path);
+ fprintf(stderr, "\tc_dbg_lvl: [%8d]\n",
+ erofs_cfg.c_dbg_lvl);
+ fprintf(stderr, "\tc_dry_run: [%8d]\n",
+ erofs_cfg.c_dry_run);
+ fprintf(stderr, "\tc_alg_name: [%8s]\n",
+ erofs_cfg.c_alg_name);
+ fprintf(stderr, "\tc_compr_maxsz: [%8d]\n",
+ erofs_cfg.c_compr_maxsz);
+ fprintf(stderr, "\tc_compr_lvl: [%8d]\n",
+ erofs_cfg.c_compr_lvl);
+ fprintf(stderr, "\tc_compr_boundary: [%8d]\n",
+ erofs_cfg.c_compr_boundary);
+ fprintf(stderr, "\tc_compr_ratio_limit: [%8d]\n",
+ erofs_cfg.c_compr_ratio_limit);
+}
+
+void mkfs_free_config(void)
+{
+ if (erofs_cfg.c_img_path)
+ free(erofs_cfg.c_img_path);
+
+ if (erofs_cfg.c_src_path)
+ free(erofs_cfg.c_src_path);
+}
diff --git a/erofs_config.h b/erofs_config.h
new file mode 100644
index 0000000..34b2dee
--- /dev/null
+++ b/erofs_config.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * mkfs_config.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_CONFIG_H
+#define __EROFS_CONFIG_H
+
+/* workaround of a lz4 native compression issue, which can crash the program */
+/* #define EROFS_CONFIG_COMPR_MAX_SZ (1024 * 1024) */
+#define EROFS_CONFIG_COMPR_MAX_SZ (900 * 1024)
+#define EROFS_CONFIG_COMPR_MIN_SZ (32 * 1024)
+#define EROFS_CONFIG_COMPR_DEF_BOUNDARY (128)
+#define EROFS_CONFIG_COMPR_RATIO_MAX_LIMIT (100)
+
+struct erofs_compr_alg;
+
+struct erofs_configure {
+ char *c_version;
+ int c_dry_run;
+ int c_dbg_lvl;
+
+ struct erofs_compr_alg *c_compr_alg;
+ int c_compr_maxsz;
+ int c_compr_lvl;
+ int c_compr_boundary;
+ int c_compr_ratio_limit;
+
+ char *c_src_path;
+ char *c_img_path;
+ char *c_label;
+ const char *c_alg_name;
+};
+
+extern struct erofs_configure erofs_cfg;
+
+void mkfs_init_configure(void);
+void mkfs_dump_config(void);
+void mkfs_free_config(void);
+
+#endif
diff --git a/erofs_debug.h b/erofs_debug.h
new file mode 100644
index 0000000..364c971
--- /dev/null
+++ b/erofs_debug.h
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_debug.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ *
+ */
+#ifndef __EROFS_PRINT_H
+#define __EROFS_PRINT_H
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/time.h>
+#include <errno.h>
+#include <string.h>
+
+#include "erofs_config.h"
+#define FUNC_LINE_FMT "%s() Line[%d]"
+
+#ifdef DEBUG
+#define ASSERT(X) assert(x)
+#else
+#define ASSERT(X) ((void)0)
+#endif
+
+#ifndef pr_fmt
+#define pr_fmt(fmt) "EROFS: "FUNC_LINE_FMT fmt"\n"
+#endif
+
+#define erofs_dbg(fmt, ...) \
+do { \
+ if (erofs_cfg.c_dbg_lvl >= 7) { \
+ fprintf(stdout, pr_fmt(fmt), __func__, \
+ __LINE__, ## __VA_ARGS__); \
+ } \
+} while (0)
+
+#define erofs_info(fmt, ...) \
+do { \
+ if (erofs_cfg.c_dbg_lvl >= 3) { \
+ fprintf(stdout, pr_fmt(fmt), __func__, \
+ __LINE__, ## __VA_ARGS__); \
+ fflush(stdout); \
+ } \
+} while (0)
+
+#define erofs_warn(fmt, ...) \
+ do { \
+ if (erofs_cfg.c_dbg_lvl >= 2) { \
+ fprintf(stdout, pr_fmt(fmt), __func__, \
+ __LINE__, ## __VA_ARGS__); \
+ fflush(stdout); \
+ } \
+ } while (0)
+
+
+#define erofs_err(fmt, ...) \
+do { \
+ if (erofs_cfg.c_dbg_lvl >= 0) { \
+ fprintf(stderr, "Err: "pr_fmt(fmt), __func__, \
+ __LINE__, ## __VA_ARGS__); \
+ } \
+} while (0)
+
+#endif
diff --git a/erofs_error.h b/erofs_error.h
new file mode 100644
index 0000000..2ce614d
--- /dev/null
+++ b/erofs_error.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_error.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_ERROR_H
+#define __EROFS_ERROR_H
+
+#define MAX_ERRNO (4095)
+#define IS_ERR_VALUE(x) \
+ ((unsigned long)(void *)(x) >= (unsigned long)-MAX_ERRNO)
+
+static inline void *ERR_PTR(long error)
+{
+ return (void *) error;
+}
+
+static inline int IS_ERR(const void *ptr)
+{
+ return IS_ERR_VALUE((unsigned long)ptr);
+}
+#endif
diff --git a/erofs_io.c b/erofs_io.c
new file mode 100644
index 0000000..4fa7608
--- /dev/null
+++ b/erofs_io.c
@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs_io.c
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#define _LARGEFILE64_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+
+#include "erofs_io.h"
+#include "mkfs_erofs.h"
+
+
+#define pr_fmt(fmt) "DEVICE IO: "FUNC_LINE_FMT fmt"\n"
+#include "erofs_debug.h"
+
+static char *erofs_devname;
+static int erofs_devfd = -1;
+static u64 erofs_devsz;
+
+void dev_close(void)
+{
+ close(erofs_devfd);
+ free(erofs_devname);
+ erofs_devname = NULL;
+ erofs_devfd = -1;
+ erofs_devsz = 0;
+}
+
+int dev_open(const char *devname)
+{
+ char *dev;
+ struct stat st;
+ int fd;
+ int ret;
+
+ dev = strdup(devname);
+ if (!dev)
+ return -ENOMEM;
+again:
+ fd = open(dev, O_RDWR);
+ if (fd < 0 && errno != ENOENT) {
+ erofs_err("Open device/file(%s) fail.", dev);
+ free(dev);
+ return -errno;
+ }
+
+ if (fd < 0) {
+ fd = open(dev, O_RDWR | O_CREAT | O_NOCTTY,
+ S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH);
+ if (fd < 0) {
+ if (errno != EEXIST) {
+ erofs_err("Create image file(%s) fail.", dev);
+ free(dev);
+ return -errno;
+ } else {
+ erofs_dbg("Image file(%s) existed, it might be created by the other users.",
+ dev);
+ goto again;
+ }
+ }
+ }
+
+ ret = fstat(fd, &st);
+ if (ret) {
+ erofs_err("Get stat of device/file(%s) fail.", dev);
+ close(fd);
+ free(dev);
+ return -errno;
+ }
+
+ if (!S_ISBLK(st.st_mode) && !S_ISREG(st.st_mode)) {
+ erofs_err("File (%s) type is wrong.", dev);
+ close(fd);
+ free(dev);
+ return -EINVAL;
+ }
+
+ if (S_ISREG(st.st_mode)) {
+ ret = ftruncate(fd, 0);
+ if (ret) {
+ erofs_err("Truncate file(%s) fail.", dev);
+ close(fd);
+ free(dev);
+ return -errno;
+ }
+ }
+
+ erofs_devname = dev;
+ erofs_devfd = fd;
+
+ if (S_ISBLK(st.st_mode)) {
+ erofs_devsz = st.st_size;
+ } else {
+ /* INT64_MAX is the limit of kernel vfs */
+ erofs_devsz = INT64_MAX;
+ }
+
+ erofs_devsz = round_down(erofs_devsz, EROFS_BLKSIZE);
+
+ erofs_info("Open device/file %s", erofs_devname);
+
+ return 0;
+}
+
+u64 dev_length(void)
+{
+ return erofs_devsz;
+}
+
+int dev_write(void *buf, u64 offset, size_t len)
+{
+ int ret;
+
+ if (erofs_cfg.c_dry_run)
+ return 0;
+
+ if (!buf) {
+ erofs_err("buf is NULL");
+ return -EINVAL;
+ }
+
+ if (offset >= erofs_devsz || len > erofs_devsz ||
+ offset > erofs_devsz - len) {
+ erofs_err("Write posion[%" PRIu64 ", %zd] is too large beyond the end of device(%" PRIu64 ").",
+ offset, len, erofs_devsz);
+ return -EINVAL;
+ }
+
+ ret = pwrite64(erofs_devfd, buf, len, (off64_t)offset);
+ if (ret != (int)len) {
+ if (ret < 0) {
+ erofs_err("Failed to write data into device - %s:[%" PRIu64 ", %zd].",
+ erofs_devname, offset, len);
+ return -errno;
+ }
+
+ erofs_err("Writing data into device - %s:[%" PRIu64 ", %zd] - was truncated.",
+ erofs_devname, offset, len);
+ return -ERANGE;
+ }
+ return 0;
+}
+
+int dev_write_block(void *buf, u32 blkaddr)
+{
+ erofs_info("Write data to block %u", blkaddr);
+
+ return dev_write(buf, ((off64_t)blkaddr) << EROFS_BLOCKSIZE_BITS,
+ EROFS_BLKSIZE);
+}
+
+int dev_fsync(void)
+{
+ int ret;
+
+ ret = fsync(erofs_devfd);
+ if (ret) {
+ erofs_err("Could not fsync device!!!");
+ return -EIO;
+ }
+ return 0;
+}
+
diff --git a/erofs_io.h b/erofs_io.h
new file mode 100644
index 0000000..b447062
--- /dev/null
+++ b/erofs_io.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_io.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_IO_H
+#define __EROFS_IO_H
+
+#include <sys/cdefs.h>
+#include "erofs_types.h"
+
+int dev_open(const char *devname);
+void dev_close(void);
+int dev_write(void *buf, u64 offset, size_t len);
+int dev_write_block(void *buf, u32 blkaddr);
+int dev_fsync(void);
+u64 dev_length(void);
+
+#endif
+
+
diff --git a/erofs_types.h b/erofs_types.h
new file mode 100644
index 0000000..6ab0a55
--- /dev/null
+++ b/erofs_types.h
@@ -0,0 +1,52 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_types.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_TYPES_H
+#define __EROFS_TYPES_H
+#include <inttypes.h>
+#include <endian.h>
+#include <linux/types.h>
+#include <sys/types.h>
+#include <asm/types.h>
+#include <stdint.h>
+#include <sys/cdefs.h>
+#include <assert.h>
+
+#define u8 uint8_t
+#define u16 uint16_t
+#define u32 uint32_t
+#define u64 uint64_t
+#define s8 int8_t
+#define s16 int16_t
+#define s32 int32_t
+#define s64 int64_t
+
+#define cpu_to_le16(X) htole16(X)
+#define cpu_to_le32(X) htole32(X)
+#define cpu_to_le64(X) htole64(X)
+
+#define le16_to_cpu(X) le16toh(X)
+#define le32_to_cpu(X) le32toh(X)
+#define le64_to_cpu(X) le64toh(X)
+
+#ifndef __OPTIMIZE__
+#define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)]))
+#else
+#define BUILD_BUG_ON(condition) assert(condition)
+#endif
+
+#define BIT(nr) (1UL << (nr))
+#define BIT_ULL(nr) (1ULL << (nr))
+#define BIT_MASK(nr) (1UL << ((nr) % BITS_PER_LONG))
+#define BIT_WORD(nr) ((nr) / BITS_PER_LONG)
+#define BIT_ULL_MASK(nr) (1ULL << ((nr) % BITS_PER_LONG_LONG))
+#define BIT_ULL_WORD(nr) ((nr) / BITS_PER_LONG_LONG)
+#define BITS_PER_BYTE 8
+#define BITS_TO_LONGS(nr) DIV_ROUND_UP(nr, BITS_PER_BYTE * sizeof(long))
+
+#endif
diff --git a/list_head.h b/list_head.h
new file mode 100644
index 0000000..fe5c891
--- /dev/null
+++ b/list_head.h
@@ -0,0 +1,110 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * list_head.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __CONFIG_LIST_H
+#define __CONFIG_LIST_H
+
+#include <assert.h>
+#include <stddef.h>
+
+
+struct list_head {
+ struct list_head *prev;
+ struct list_head *next;
+};
+
+#define container_of(ptr, type, member) ({ \
+ const typeof(((type *)0)->member) * __mptr = (ptr); \
+ (type *)((char *)__mptr - offsetof(type, member)); })
+
+
+#define LIST_HEAD_INIT(name) {&(name), &(name)}
+
+#define LIST_HEAD(name) \
+ struct list_head name = LIST_HEAD_INIT(name)
+
+static inline void init_list_head(struct list_head *list)
+{
+ list->prev = list;
+ list->next = list;
+}
+
+static inline void __list_add(struct list_head *entry,
+ struct list_head *prev,
+ struct list_head *next)
+{
+ entry->prev = prev;
+ entry->next = next;
+ prev->next = entry;
+ next->prev = entry;
+}
+
+static inline void list_add(struct list_head *entry, struct list_head *head)
+{
+ __list_add(entry, head, head->next);
+}
+
+static inline void list_add_tail(struct list_head *entry,
+ struct list_head *head)
+{
+ __list_add(entry, head->prev, head);
+}
+
+static inline void __list_del(struct list_head *prev, struct list_head *next)
+{
+ prev->next = next;
+ next->prev = prev;
+}
+
+static inline void list_del(struct list_head *entry)
+{
+ __list_del(entry->prev, entry->next);
+ entry->prev = entry->next = NULL;
+}
+
+static inline int list_empty(struct list_head *head)
+{
+ return head->next == head;
+}
+
+#define list_entry(ptr, type, member) \
+ container_of(ptr, type, member)
+
+#define list_first_entry(ptr, type, member) \
+ list_entry((ptr)->next, type, member)
+
+#define list_next_entry(pos, member) \
+ list_entry((pos)->member.next, typeof(*(pos)), member)
+
+#define list_prev_entry(pos, member) \
+ list_entry((pos)->member.prev, typeof(*(pos)), member)
+
+#define list_for_each(pos, head) \
+ for (pos = (head)->next; pos != (head); pos = pos->next)
+
+#define list_for_each_safe(pos, n, head) \
+ for (pos = (head)->next, n = pos->next; \
+ pos != (head); \
+ pos = n, n = pos->next)
+
+#define list_for_each_entry(pos, head, member) \
+ for (pos = list_first_entry(head, typeof(*pos), member); \
+ &pos->member != (head); \
+ pos = list_next_entry(pos, member))
+
+#define list_for_each_entry_from(pos, head, member) \
+ for (; &pos->member != (head); \
+ pos = list_next_entry(pos, member))
+
+#define list_for_each_entry_safe(pos, n, head, member) \
+ for (pos = list_first_entry(head, typeof(*pos), member), \
+ n = list_next_entry(pos, member); \
+ &pos->member != (head); \
+ pos = n, n = list_next_entry(n, member))
+
+#endif
diff --git a/mkfs_erofs.h b/mkfs_erofs.h
new file mode 100644
index 0000000..be6305e
--- /dev/null
+++ b/mkfs_erofs.h
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * mkfs_erofs.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ *
+ */
+#ifndef __EROFS_MKFS_H
+#define __EROFS_MKFS_H
+#include <linux/limits.h>
+#include "list_head.h"
+#include "erofs_types.h"
+
+typedef unsigned int __u32;
+
+#define __packed __attribute__((__packed__))
+
+#include "erofs_fs.h"
+
+#ifndef PAGE_SIZE
+#define PAGE_SIZE (4096)
+#endif
+
+#ifndef EROFS_BLKSIZE
+#define EROFS_BLKSIZE (4096)
+#define EROFS_BLOCKSIZE_BITS (12)
+#endif
+
+#define EROFS_BLOCK_SIZE_SHIFT_BITS (12)
+#define EROFS_SLOTSIZE_BITS (5)
+#define EROFS_SLOTSIZE (32)
+#define MKFS_DIFF_SHIFT_8_BITS (8)
+
+#define __round_mask(x, y) ((__typeof__(x))((y)-1))
+#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
+#define round_down(x, y) ((x) & ~__round_mask(x, y))
+
+#define SIZE_ALIGN(val, size) (((val) + (size) - 1) & (~(size-1)))
+#define SLOT_ALIGN(slots) SIZE_ALIGN(slots, EROFS_SLOTSIZE)
+#define PAGE_ALIGN(pages) SIZE_ALIGN(pages, PAGE_SIZE)
+#define BLK_ALIGN(blks) SIZE_ALIGN(blks, EROFS_BLKSIZE)
+#define IS_SLOT_ALIGN(__ADDR) (((__ADDR)%(EROFS_SLOTSIZE))?0:1)
+#define IS_BLK_ALIGN(__ADDR) (((__ADDR)%(EROFS_BLKSIZE))?0:1)
+#define ADDR_TO_BLKNO(__ADDR) ((__ADDR) >> EROFS_BLOCKSIZE_BITS)
+#define BLKNO_TO_ADDR(__ADDR) ((u64)(__ADDR) << EROFS_BLOCKSIZE_BITS)
+#define MAX_NID_INDEX_PER_BLK (EROFS_BLKSIZE / EROFS_SLOTSIZE)
+
+#define EROFS_INODE_V1_SIZE sizeof(struct erofs_inode_v1)
+#define EROFS_INODE_V2_SIZE sizeof(struct erofs_inode_v2)
+
+#define EROFS_DIRENT_SIZE sizeof(struct erofs_dirent)
+
+#define EROFS_DECOMPR_IDX_SZ sizeof(struct z_erofs_vle_decompressed_index)
+#define EROFS_DECOMPR_IDXS_PER_BLK (EROFS_BLKSIZE / EROFS_DECOMPR_IDX_SZ)
+
+#define ondisk_extent_size(data_mapping_mode, count) \
+ ((data_mapping_mode) == EROFS_INODE_LAYOUT_COMPRESSION ? \
+ (sizeof(struct erofs_extent_header) + \
+ sizeof(__u32) * le32_to_cpu(count)) : 0)
+
+#define EROFS_INLINE_GENERIC_ALIGN_SIZE (4)
+#define EROFS_INLINE_INDEX_ALIGN_SIZE (8)
+
+#endif
diff --git a/mkfs_main.c b/mkfs_main.c
new file mode 100644
index 0000000..ae92656
--- /dev/null
+++ b/mkfs_main.c
@@ -0,0 +1,165 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * mkfs_main.c
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#include <stdio.h>
+#include <string.h>
+#include <sys/time.h>
+#include <time.h>
+#include <linux/fs.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <libgen.h>
+#include "mkfs_erofs.h"
+#include "erofs_io.h"
+#define pr_fmt(fmt) "MKFS: "FUNC_LINE_FMT fmt"\n"
+#include "erofs_debug.h"
+
+
+#define EROFS_SUPER_END (EROFS_SUPER_OFFSET + sizeof(struct erofs_super_block))
+#define ROOT_INODE_NUM (0) /*always 0 match at rank inode */
+#define EROFS_META_BLK_ADDR_DEFAULT (1)
+#define EROFS_XATTR_BLK_ADDR_DEFAULT (0)
+
+static struct erofs_super_block erosb = {
+ .magic = cpu_to_le32(EROFS_SUPER_MAGIC_V1),
+ .blkszbits = EROFS_BLOCK_SIZE_SHIFT_BITS,
+ .root_nid = cpu_to_le16(ROOT_INODE_NUM),
+ .inos = 0,
+ .blocks = 0,
+ .meta_blkaddr = cpu_to_le32(EROFS_META_BLK_ADDR_DEFAULT),
+ .xattr_blkaddr = cpu_to_le32(EROFS_XATTR_BLK_ADDR_DEFAULT),
+
+};
+struct erofs_super_block *sb = &erosb;
+
+
+static void usage(char *path)
+{
+ char *cmd = realpath(path, NULL);
+
+ fprintf(stderr, "%s %s\n", basename(cmd), erofs_cfg.c_version);
+ fprintf(stderr, "\nUsage:\n");
+ fprintf(stderr, " [-z <compr_algri>] [-d <dbglvl>]\n");
+ fprintf(stderr, " [target path] [source directory]\n");
+ exit(1);
+}
+
+u64 parse_num_from_str(const char *str)
+{
+ u64 num = 0;
+ char *endptr = NULL;
+
+ num = strtoull(str, &endptr, 10);
+ ASSERT(num != ULLONG_MAX);
+ return num;
+}
+
+static void mkfs_parse_options_cfg(int argc, char *argv[])
+{
+ int opt;
+
+ while ((opt = getopt(argc, argv, "d:h:z::x")) != -1) {
+ switch (opt) {
+ case 'z':
+ if (optarg)
+ erofs_cfg.c_alg_name = strdup(optarg);
+ else
+ erofs_cfg.c_alg_name = "lz4hc";
+ break;
+
+ case 'd':
+ erofs_cfg.c_dbg_lvl = parse_num_from_str(optarg);
+ break;
+
+ default: /* '?' */
+ usage(argv[0]);
+ }
+ }
+
+ if (optind >= argc)
+ usage(argv[0]);
+
+ erofs_cfg.c_img_path = strdup(argv[optind++]);
+ assert(erofs_cfg.c_img_path);
+
+ if (optind < argc) {
+ erofs_cfg.c_src_path = strdup(argv[optind++]);
+ erofs_cfg.c_src_path = realpath(erofs_cfg.c_src_path, NULL);
+ if (!erofs_cfg.c_src_path) {
+ perror("c_src_path realpath");
+ usage(argv[0]);
+ }
+ } else {
+ erofs_err("c_src_path is NULL");
+ usage(argv[0]);
+ }
+ assert(erofs_cfg.c_src_path);
+
+ if (optind < argc) {
+ erofs_err(" Unexpected argument: %s\n", argv[optind]);
+ usage(argv[0]);
+ }
+
+ mkfs_dump_config();
+
+ if (dev_open(erofs_cfg.c_img_path) < 0) {
+ erofs_err("dev_open is failed!!!");
+ usage(argv[0]);
+ }
+}
+
+void mkfs_update_erofs_header(u64 root_addr)
+{
+ int ret = 0;
+ u64 size = 0;
+ char *sb_buf = NULL;
+ struct timeval t;
+
+ size = BLK_ALIGN(EROFS_SUPER_END);
+
+ if (gettimeofday(&t, NULL) == 0) {
+ sb->build_time = cpu_to_le64(t.tv_sec);
+ sb->build_time_nsec = cpu_to_le32(t.tv_usec);
+ }
+
+ sb->meta_blkaddr = cpu_to_le32(ADDR_TO_BLKNO(size));
+
+ sb_buf = calloc(size, 1);
+ if (!sb_buf) {
+ perror("calloc");
+ erofs_err("\tError: Failed to calloc!!!\n");
+ exit(EXIT_FAILURE);
+ }
+
+ memcpy(sb_buf + EROFS_SUPER_OFFSET, sb, sizeof(*sb));
+
+ ret = dev_write(sb_buf, 0, size);
+ if (ret < 0) {
+ erofs_err("dev_write failed ret=%d", ret);
+ exit(EXIT_FAILURE);
+ }
+}
+
+int main(int argc, char **argv)
+{
+ int err = 0;
+ struct erofs_node_info *proot_node = NULL;
+
+ mkfs_init_configure();
+ mkfs_parse_options_cfg(argc, argv);
+
+ mkfs_update_erofs_header(proot_node->i_base_addr);
+
+ dev_close();
+ mkfs_free_config();
+ erofs_info("done");
+
+ return err;
+}
+
--
2.17.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [PREVIEW] [RFC PATCH v2 3/5] erofs-mkfs: support to build a image from the specific root directory
2018-11-20 14:32 [PREVIEW] [RFC PATCH v2 0/5] erofs-mkfs: mkfs.erofs source code first preview Li Guifu
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 1/5] erofs-mkfs: add erofs on-disk layout Li Guifu
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 2/5] erofs-mkfs: introduce mkfs basic framework Li Guifu
@ 2018-11-20 14:32 ` Li Guifu
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 4/5] erofs-mkfs: add compression support Li Guifu
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 5/5] erofs-mkfs: add miscellaneous files Li Guifu
4 siblings, 0 replies; 6+ messages in thread
From: Li Guifu @ 2018-11-20 14:32 UTC (permalink / raw)
Currently, regular files, directories, special files defined in POSIX
are supported and the generated image can be mounted by kernel.
Signed-off-by: Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Fang Wei <fangwei1 at huawei.com>
---
erofs_cache.c | 212 ++++++++++++++
erofs_cache.h | 58 ++++
mkfs_file.c | 306 ++++++++++++++++++++
mkfs_file.h | 79 +++++
mkfs_inode.c | 786 ++++++++++++++++++++++++++++++++++++++++++++++++++
mkfs_inode.h | 84 ++++++
mkfs_main.c | 28 ++
7 files changed, 1553 insertions(+)
create mode 100644 erofs_cache.c
create mode 100644 erofs_cache.h
create mode 100644 mkfs_file.c
create mode 100644 mkfs_file.h
create mode 100644 mkfs_inode.c
create mode 100644 mkfs_inode.h
diff --git a/erofs_cache.c b/erofs_cache.c
new file mode 100644
index 0000000..18c2390
--- /dev/null
+++ b/erofs_cache.c
@@ -0,0 +1,212 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs_cache.c
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Miao Xie <miaoxie at huawei.com>
+ */
+#include <string.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <errno.h>
+
+#include "erofs_types.h"
+#include "erofs_cache.h"
+#include "erofs_error.h"
+#include "erofs_debug.h"
+#include "erofs_io.h"
+#include "mkfs_erofs.h"
+#include "mkfs_inode.h"
+
+/* The 1st block is used for SUPER BLOCK, so skip it. */
+static u64 erofs_current_block = 1;
+static LIST_HEAD(erofs_global_blocks);
+static LIST_HEAD(erofs_free_blocks);
+static LIST_HEAD(erofs_full_blocks);
+
+static char *erofs_blk_buf;
+
+/*
+ * Note: Since we reserved the 1st block for super block, so we can reuse
+ * this block number as a special number, we use it to tell the caller that
+ * there is NO ENOUGH FREE SPACE!!!!
+ *
+ * So the result value:
+ * !0 : Allocate the blocks successfully, this is block number of
+ * the 1st block.
+ * 0 : ENOSPC, There is no enough space.
+ */
+u32 erofs_alloc_blocks(u32 nblocks)
+{
+ u32 blkaddr;
+ u64 devlen;
+
+ assert(nblocks);
+
+ devlen = dev_length();
+ if (erofs_current_block > (u64)UINT32_MAX ||
+ erofs_current_block + nblocks > ((u64)UINT32_MAX) + 1 ||
+ (erofs_current_block + nblocks) << EROFS_BLOCKSIZE_BITS > devlen) {
+ erofs_err("There is no enough free space(curr: %llu, need: %u, device blocks: %llu).",
+ (unsigned long long)erofs_current_block, nblocks,
+ (unsigned long long)devlen >> EROFS_BLOCKSIZE_BITS);
+ return 0;
+ }
+
+ blkaddr = (u32)erofs_current_block;
+ erofs_current_block += nblocks;
+
+ return blkaddr;
+}
+
+u64 erofs_get_total_blocks(void)
+{
+ return erofs_current_block;
+}
+
+
+block_buffer_t *erofs_alloc_multi_block_buffer(u32 nblocks)
+{
+ block_buffer_t *blk;
+ block_buffer_t *next;
+ block_buffer_t *first;
+ struct list_head blocks;
+ u32 blkaddr;
+ u32 i;
+ int ret;
+
+ init_list_head(&blocks);
+
+ for (i = 0; i < nblocks; i++) {
+ blk = (block_buffer_t *)malloc(sizeof(block_buffer_t));
+ if (!blk) {
+ erofs_err("Fail to alloc memory for block buffer");
+ ret = -ENOMEM;
+ goto free_block_buffer;
+ }
+ memset(blk, 0, sizeof(block_buffer_t));
+ init_list_head(&blk->bb_metadata_list);
+
+ list_add_tail(&blk->bb_global_node, &blocks);
+ }
+
+ blkaddr = erofs_alloc_blocks(nblocks);
+ if (!blkaddr) {
+ ret = -ENOSPC;
+ goto free_block_buffer;
+ }
+
+ first = list_first_entry(&blocks, block_buffer_t, bb_global_node);
+ list_for_each_entry_safe(blk, next, &blocks, bb_global_node) {
+ blk->bb_blkaddr = blkaddr;
+ blkaddr++;
+
+ list_del(&blk->bb_global_node);
+ list_add_tail(&blk->bb_global_node, &erofs_global_blocks);
+ list_add_tail(&blk->bb_alloc_node, &erofs_free_blocks);
+ }
+
+ return first;
+
+free_block_buffer:
+ list_for_each_entry_safe(blk, next, &blocks, bb_global_node) {
+ list_del(&blk->bb_global_node);
+ free(blk);
+ }
+ return ERR_PTR(ret);
+}
+
+block_buffer_t *erofs_alloc_single_block_buffer(void)
+{
+ return erofs_alloc_multi_block_buffer(1);
+}
+
+block_buffer_t *erofs_look_up_free_pos(int request_size)
+{
+ block_buffer_t *blk = NULL;
+
+ list_for_each_entry(blk, &erofs_free_blocks, bb_alloc_node) {
+ if ((request_size + blk->bb_free_slot * EROFS_SLOTSIZE) <
+ EROFS_BLKSIZE)
+ return blk;
+ }
+
+ blk = erofs_alloc_single_block_buffer();
+ return blk;
+}
+
+int erofs_flush_all_blocks(void)
+{
+ struct block_buffer *blk;
+ struct erofs_meta_node *node;
+ struct erofs_node_info *inode;
+ struct erofs_index_info *index;
+ char *pbuf;
+ int count;
+ int ret;
+ u32 addr;
+
+ list_for_each_entry(blk, &erofs_global_blocks, bb_global_node) {
+ pbuf = erofs_blk_buf;
+ memset(pbuf, 0, EROFS_BLKSIZE);
+
+ list_for_each_entry(node, &blk->bb_metadata_list, m_node) {
+ switch (node->m_type) {
+ case EROFS_META_INODE:
+ inode = (struct erofs_node_info *)node;
+
+ count = erofs_write_inode_buffer(inode, pbuf);
+ break;
+ case EROFS_META_INDEX:
+ index = (struct erofs_index_info *)node;
+
+ count = erofs_write_index_buffer(index, pbuf);
+ break;
+ default:
+ erofs_err("Wrong metadata type");
+ assert(0);
+ }
+
+ count = round_up(count, EROFS_SLOTSIZE);
+ assert(count == node->m_len);
+ pbuf += count;
+ }
+
+ addr = blk->bb_blkaddr;
+
+ ret = dev_write(erofs_blk_buf, BLKNO_TO_ADDR(addr),
+ EROFS_BLKSIZE);
+ if (ret)
+ break;
+ }
+
+ return ret;
+}
+
+void erofs_put_block_buffer(struct block_buffer *blk)
+{
+ if (blk->bb_free_slot == MAX_NID_INDEX_PER_BLK) {
+ list_del(&blk->bb_alloc_node);
+ list_add_tail(&blk->bb_alloc_node, &erofs_full_blocks);
+ } else if (blk->bb_free_slot > MAX_NID_INDEX_PER_BLK) {
+ erofs_err("block buffer overflow: free_slot = %d, MAX_NID_INDEX_PER_BLK = %d",
+ blk->bb_free_slot, MAX_NID_INDEX_PER_BLK);
+ assert(0);
+ }
+}
+
+int erofs_cache_init(u64 start_blk)
+{
+ erofs_blk_buf = malloc(EROFS_BLKSIZE);
+ if (!erofs_blk_buf)
+ return -ENOMEM;
+ erofs_current_block = start_blk;
+ return 0;
+}
+
+void erofs_cache_deinit(void)
+{
+ if (erofs_blk_buf)
+ free(erofs_blk_buf);
+}
diff --git a/erofs_cache.h b/erofs_cache.h
new file mode 100644
index 0000000..15d9c04
--- /dev/null
+++ b/erofs_cache.h
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_cache.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_CACHE_H__
+#define __EROFS_CACHE_H__
+#include "mkfs_erofs.h"
+#include "list_head.h"
+
+enum erofs_meta_type {
+ EROFS_META_INODE,
+ EROFS_META_INDEX,
+ EROFS_META_MAX_TYPES,
+};
+
+struct erofs_meta_node {
+ struct list_head m_node;
+ struct block_buffer *m_blk;
+ int m_type;
+ int m_slot;
+ int m_len;
+};
+
+static inline void erofs_meta_node_init(struct erofs_meta_node *node, int type)
+{
+ init_list_head(&node->m_node);
+ node->m_type = type;
+ node->m_blk = NULL;
+ node->m_slot = 0;
+ node->m_len = 0;
+}
+
+typedef struct block_buffer {
+ /* These two members are used to block management. */
+ struct list_head bb_global_node;
+ struct list_head bb_alloc_node;
+
+ /* This is the head of all metadata which is hold in this block. */
+ struct list_head bb_metadata_list;
+
+ u32 bb_blkaddr;
+ int bb_free_slot;
+} block_buffer_t;
+
+block_buffer_t *erofs_alloc_single_block_buffer(void);
+block_buffer_t *erofs_alloc_multi_block_buffer(u32 nblocks);
+block_buffer_t *erofs_look_up_free_pos(int request_size);
+int erofs_flush_all_blocks(void);
+int erofs_cache_init(u64 start_blk);
+void erofs_cache_deinit(void);
+u32 erofs_alloc_blocks(u32 nblocks);
+void erofs_put_block_buffer(struct block_buffer *blk);
+u64 erofs_get_total_blocks(void);
+#endif
diff --git a/mkfs_file.c b/mkfs_file.c
new file mode 100644
index 0000000..beeb89d
--- /dev/null
+++ b/mkfs_file.c
@@ -0,0 +1,306 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * mkfs_file.c
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#define _LARGEFILE64_SOURCE
+#include <assert.h>
+#include <libgen.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <dirent.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <linux/kdev_t.h>
+
+
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+#include "erofs_types.h"
+#include "list_head.h"
+#include "erofs_cache.h"
+
+#define pr_fmt(fmt) "MKFS-FILE: "FUNC_LINE_FMT fmt"\n"
+#include "erofs_debug.h"
+
+#include "mkfs_erofs.h"
+#include "mkfs_file.h"
+#include "mkfs_inode.h"
+#include "erofs_io.h"
+
+
+#define DIRENT_MAX_NAME_LEN 256
+
+
+static u8 get_file_type(struct stat64 *st)
+{
+ u8 file_type = EROFS_FT_MAX;
+
+ switch (st->st_mode & S_IFMT) {
+ case S_IFREG:
+ file_type = EROFS_FT_REG_FILE;
+ break;
+
+ case S_IFDIR:
+ file_type = EROFS_FT_DIR;
+ break;
+
+ case S_IFLNK:
+ file_type = EROFS_FT_SYMLINK;
+ break;
+
+ case S_IFCHR:
+ file_type = EROFS_FT_CHRDEV;
+ break;
+
+ case S_IFBLK:
+ file_type = EROFS_FT_BLKDEV;
+ break;
+
+ case S_IFIFO:
+ file_type = EROFS_FT_FIFO;
+ break;
+
+ case S_IFSOCK:
+ file_type = EROFS_FT_SOCK;
+ break;
+
+ default:
+ erofs_err("file type[0x%X]", st->st_mode & S_IFMT);
+ break;
+ }
+
+ return file_type;
+
+}
+
+static inline u32 new_encode_dev(dev_t dev)
+{
+ unsigned int major = MAJOR(dev);
+ unsigned int minor = MINOR(dev);
+
+ return (minor & 0xff) | (major << 8) | ((minor & ~0xff) << 12);
+}
+
+
+struct erofs_node_info *erofs_init_inode(char *full_path_name)
+{
+ int ret;
+ struct stat64 st;
+ struct erofs_node_info *inode = NULL;
+ char *file_name = NULL;
+
+ file_name = strrchr(full_path_name, '/');
+ if (!file_name) {
+ file_name = full_path_name;
+ } else {
+ file_name = file_name + 1;
+ }
+
+ inode = alloc_erofs_node();
+ if (!inode) {
+ erofs_err("inode is NULL, alloc failed");
+ goto Err_alloc;
+ }
+
+ ret = snprintf(inode->i_name, MAX_NAME, "%s", file_name);
+ if (ret < 0 || ret >= MAX_PATH) {
+ erofs_err("snprintf errorly file_name[%s] ret[%d]",
+ file_name, ret);
+ goto Err_alloced;
+ }
+ ret = snprintf(inode->i_fullpath, MAX_PATH, "%s", full_path_name);
+ if (ret < 0 || ret >= MAX_PATH) {
+ erofs_err("snprintf errorly full_path_name[%s] ret[%d]",
+ full_path_name, ret);
+ goto Err_alloced;
+ }
+
+ ret = lstat64(inode->i_fullpath, &st);
+ if (ret) {
+ erofs_err("stat failed path[%s]", inode->i_fullpath);
+ goto Err_alloced;
+ }
+
+ /* It is ugly code that is for old code everywhere */
+ inode->i_mode = st.st_mode;
+ inode->i_uid = st.st_uid;
+ inode->i_gid = st.st_gid;
+ inode->i_nlink = st.st_nlink;
+ inode->i_type = get_file_type(&st);
+
+ if (S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode) ||
+ S_ISFIFO(st.st_mode) || S_ISSOCK(st.st_mode)) {
+ inode->i_rdev = new_encode_dev(st.st_rdev);
+ inode->i_size = 0;
+ } else {
+ inode->i_size = st.st_size;
+ }
+
+ return inode;
+
+
+Err_alloced:
+ free(inode);
+
+Err_alloc:
+ return NULL;
+}
+
+int erofs_create_files_list(struct erofs_node_info *inode)
+{
+ int ret = 0;
+ u64 d_size = 0;
+ DIR *dirp = NULL;
+ char file_path[MAX_PATH + 1];
+ struct stat64 s;
+ struct dirent *dp;
+ struct list_head *pos;
+ struct erofs_node_info *dl;
+
+
+ if (!strncmp(inode->i_name, "lost+found", strlen("lost+found"))) {
+ return 0;
+ }
+
+ if (lstat64(inode->i_fullpath, &s) == 0) {
+ if (S_ISREG(s.st_mode)) {
+ erofs_err("[%s] is a regular file",
+ inode->i_fullpath);
+ ret = -ENOTDIR;
+ goto error;
+ }
+ } else {
+ erofs_err("stat failed [%s]", inode->i_fullpath);
+ ret = -ENOENT;
+ goto error;
+ }
+
+ dirp = opendir(inode->i_fullpath);
+ if (!dirp) {
+ erofs_info("dirp is NULL dir=%s errno=%s",
+ inode->i_fullpath, strerror(errno));
+ ret = -errno;
+ goto error;
+ }
+
+ errno = 0;
+ while ((dp = readdir(dirp)) != NULL) {
+ if (!strcmp(dp->d_name, ".") || !strcmp(dp->d_name, ".."))
+ continue;
+
+ ret = snprintf(file_path, MAX_PATH, "%s/%s",
+ inode->i_fullpath, dp->d_name);
+ if (ret < 0 || ret >= MAX_PATH) {
+ erofs_err("snprintf errorly ret[%d]", ret);
+ ret = -ENOMEM;
+ goto error;
+ }
+ dl = erofs_init_inode(file_path);
+ if (!dl) {
+ erofs_err("init inode failed !!");
+ ret = -ENOENT;
+ goto error;
+ }
+
+ dl->i_iver = erofs_check_disk_inode_version(dl);
+ list_add_sort(&inode->i_subdir_head, dl);
+ }
+
+ if (errno != 0) {
+ erofs_err("inode[%s] error[%s]",
+ inode->i_name, strerror(EBADF));
+ ret = -errno;
+ goto error;
+ }
+
+ list_for_each(pos, &inode->i_subdir_head) {
+ struct erofs_node_info *d = container_of(pos,
+ struct erofs_node_info,
+ i_list);
+ if (((d_size & (EROFS_BLKSIZE - 1)) +
+ EROFS_DIRENT_SIZE +
+ strlen(d->i_name)) > EROFS_BLKSIZE) {
+ d_size = round_up(d_size, EROFS_BLKSIZE);
+ }
+ d_size += EROFS_DIRENT_SIZE + strlen(d->i_name);
+ }
+ inode->i_size = d_size;
+
+ list_for_each(pos, &inode->i_subdir_head) {
+ struct erofs_node_info *d = container_of(pos,
+ struct erofs_node_info,
+ i_list);
+ if (d->i_type == EROFS_FT_DIR) {
+ ret = erofs_create_files_list(d);
+ if (ret < 0)
+ goto error;
+ }
+ }
+
+ closedir(dirp);
+ return 0;
+error:
+ return ret;
+}
+
+
+int list_add_sort(struct list_head *head, struct erofs_node_info *inode)
+{
+ struct list_head *pos;
+
+ if (list_empty(head)) {
+ list_add(&inode->i_list, head);
+ return 0;
+ }
+
+ list_for_each(pos, head) {
+ struct erofs_node_info *d = container_of(pos,
+ struct erofs_node_info,
+ i_list);
+
+ if (strcmp(d->i_name, inode->i_name) <= 0)
+ continue;
+
+ list_add_tail(&inode->i_list, &d->i_list);
+ return 0;
+ }
+
+ list_add_tail(&inode->i_list, head);
+ return 0;
+
+}
+
+struct erofs_node_info *alloc_erofs_node(void)
+{
+ struct erofs_node_info *f = calloc(sizeof(struct erofs_node_info), 1);
+
+ if (!f) {
+ erofs_err("calloc failed!!!");
+ return NULL;
+ }
+
+ f->i_inline_align_size = EROFS_INLINE_GENERIC_ALIGN_SIZE;
+ erofs_meta_node_init(&f->i_meta_node, EROFS_META_INODE);
+ init_list_head(&f->i_subdir_head);
+ init_list_head(&f->i_compr_idxs_list);
+ init_list_head(&f->i_xattr_head);
+
+ return f;
diff --git a/mkfs_file.h b/mkfs_file.h
new file mode 100644
index 0000000..dca5093
--- /dev/null
+++ b/mkfs_file.h
@@ -0,0 +1,79 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * mkfs_file.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __MKFS_FILE_H
+#define __MKFS_FILE_H
+
+#include <assert.h>
+#include <stdint.h>
+#include <sys/types.h>
+#include <stdio.h>
+#include <unistd.h>
+#include "list_head.h"
+#include "mkfs_erofs.h"
+
+#define FILE_TYPE_NONE (0)
+#define FILE_TYPE_COMPR (1)
+#define FILE_TYPE_MAX (2)
+
+/*
+ * Some micros for generic compress index conversion:
+ * CIG: Compress Index Generic
+ * D0: Delta0
+ * D1: Delta1
+ * LO: low 4 bits
+ * HI: high 4 bits
+ */
+#define EROFS_CIG_D0_LO_MASK (0xF)
+#define EROFS_CIG_D0_LO_SHL (12)
+#define EROFS_CIG_D0_HI_MASK (0xF0)
+#define EROFS_CIG_D0_HI_SHR (0)
+#define EROFS_CIG_D1_SHIFT (8)
+
+struct erofs_compr_idx {
+ u16 di_advise;
+ u16 di_clusterofs;
+ u16 delta[2]; /* [0] - relative index to the 1st block */
+ /* [1] - relative index to the last block */
+ u32 blkaddr;
+};
+
+/* cc_nidxs == -1 means it is a inlined compress data */
+#define EROFS_COMPR_CTX_INLINED_DATA (-1)
+
+struct erofs_compr_ctx {
+ char *cc_srcbuf;
+ char *cc_dstbuf;
+ u64 cc_pos;
+ int cc_buflen;
+ int cc_srclen;
+ int cc_dstlen;
+ struct erofs_compr_idx *cc_idxs;
+ int cc_nidxs;
+};
+
+struct erofs_compr_info {
+ struct erofs_compr_alg *ci_alg;
+ int ci_lvl;
+};
+
+struct erofs_node_info;
+
+struct erofs_node_info *alloc_erofs_node(void);
+struct erofs_node_info *erofs_init_inode(char *full_path_name);
+int list_add_sort(struct list_head *head, struct erofs_node_info *entry);
+void dump_inode(struct erofs_inode_v1 *inode);
+int erofs_create_files_list(struct erofs_node_info *entry);
+u32 erofs_calc_inline_data_size(struct erofs_node_info *inode);
+int erofs_check_compressible(struct erofs_node_info *inode);
+int erofs_compress_file(struct erofs_node_info *inode);
+void erofs_dump_compr_radio(void);
+int erofs_init_compress_context(struct erofs_compr_ctx *ctx);
+void erofs_deinit_compress_context(struct erofs_compr_ctx *ctx);
+void erofs_reset_compress_context(struct erofs_compr_ctx *ctx);
+#endif
\ No newline at end of file
diff --git a/mkfs_inode.c b/mkfs_inode.c
new file mode 100644
index 0000000..e190853
--- /dev/null
+++ b/mkfs_inode.c
@@ -0,0 +1,786 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * mkfs_inode.c
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <inttypes.h>
+#include <libgen.h>
+
+#include "erofs_cache.h"
+#include "erofs_error.h"
+#include "mkfs_erofs.h"
+#include "mkfs_file.h"
+#include "mkfs_inode.h"
+#include "erofs_io.h"
+
+
+#define pr_fmt(fmt) "MKFS_INODE: "FUNC_LINE_FMT fmt"\n"
+#include "erofs_debug.h"
+
+extern struct erofs_super_block *sb;
+
+u32 erofs_calc_inode_base_size(struct erofs_node_info *inode)
+{
+ u32 size;
+
+ if (inode->i_iver == EROFS_INODE_LAYOUT_V1)
+ size = EROFS_INODE_V1_SIZE;
+ else
+ size = EROFS_INODE_V2_SIZE;
+
+ return size;
+}
+
+u32 erofs_calc_inline_data_size(struct erofs_node_info *inode)
+{
+ u32 size = erofs_calc_inode_base_size(inode);
+
+ if (size >= EROFS_BLKSIZE)
+ return 0;
+ else
+ return (EROFS_BLKSIZE - size);
+}
+
+static inline u64 erofs_calc_compr_index_count(struct erofs_node_info *inode)
+{
+ return round_up(inode->i_size, EROFS_BLKSIZE) / EROFS_BLKSIZE;
+}
+
+u8 erofs_check_disk_inode_version(struct erofs_node_info *inode)
+{
+#if 1
+ (void)inode;
+ return EROFS_INODE_LAYOUT_V1;
+#else
+ /*
+ * Check if the members of v0 inode structure can hold the data,
+ * Check Item:
+ * - i_size: 32bits vs 64 bits
+ * - i_uid: 16bits vs 32bits
+ * - i_gid: 16bits vs 32bits
+ * - i_nlink:16bits vs 32bits
+ * - i_ctime:If it is set or not
+ */
+#endif
+}
+
+void mkfs_rank_inode(struct erofs_node_info *inode)
+{
+ block_buffer_t *blk;
+ block_buffer_t *next;
+ struct erofs_meta_node *node;
+ struct erofs_index_info *indexes;
+ u64 request_size;
+ u64 noninline_nidxs;
+ u64 first_idx;
+ u64 idx_nblks;
+ int nidxs;
+ int inline_size = 0;
+
+ node = &inode->i_meta_node;
+ request_size = erofs_calc_inode_base_size(inode);
+
+ request_size = round_up(request_size,
+ inode->i_inline_align_size);
+
+ if (inode->i_dmode == EROFS_INODE_LAYOUT_INLINE) {
+ inline_size = inode->i_size % EROFS_BLKSIZE;
+ /* we put inode into inline mode asasp */
+ if (inline_size + request_size > EROFS_BLKSIZE) {
+ erofs_err("inode[%s] inline data overflow i_size=%u slots size=%d",
+ inode->i_name, inode->i_dmode, inline_size);
+ assert(0);
+ }
+
+
+ } else if (inode->i_dmode == EROFS_INODE_LAYOUT_COMPRESSION) {
+ inline_size += inode->i_inline_datalen;
+
+ /* we put inode into inline mode asasp */
+ if (inline_size + request_size > EROFS_BLKSIZE) {
+ erofs_err("inode[%s] inline data overflow i_size=%" PRIu64 " slots size=%d",
+ inode->i_name, inode->i_size, inline_size);
+ assert(0);
+ }
+ }
+ request_size += inline_size;
+
+ /*
+ * If no compress indexes or all indexes are inlined, noninline_nidxs
+ * should be zero.
+ */
+ noninline_nidxs = inode->i_compr_nidxs - inode->i_compr_inlined_nidxs;
+ idx_nblks = noninline_nidxs * EROFS_DECOMPR_IDX_SZ;
+ idx_nblks = round_up(idx_nblks, EROFS_BLKSIZE) / EROFS_BLKSIZE;
+
+ if (idx_nblks) {
+ assert(request_size == EROFS_BLKSIZE);
+
+ blk = erofs_alloc_multi_block_buffer(idx_nblks + 1);
+ } else {
+ blk = erofs_look_up_free_pos(request_size);
+ }
+
+ if (IS_ERR(blk))
+ assert(0);
+
+ node->m_blk = blk;
+ node->m_slot = blk->bb_free_slot;
+ node->m_len = round_up(request_size, EROFS_SLOTSIZE);
+ list_add_tail(&node->m_node, &blk->bb_metadata_list);
+ inode->i_base_addr = BLKNO_TO_ADDR(blk->bb_blkaddr) +
+ blk->bb_free_slot * EROFS_SLOTSIZE;
+ blk->bb_free_slot += node->m_len / EROFS_SLOTSIZE;
+ next = list_next_entry(blk, bb_global_node);
+ erofs_put_block_buffer(blk);
+ blk = next;
+ first_idx = inode->i_compr_inlined_nidxs;
+
+ while (noninline_nidxs) {
+ if (noninline_nidxs >= EROFS_DECOMPR_IDXS_PER_BLK) {
+ nidxs = EROFS_DECOMPR_IDXS_PER_BLK;
+ request_size = EROFS_BLKSIZE;
+ } else {
+ nidxs = noninline_nidxs;
+ request_size = nidxs * EROFS_DECOMPR_IDX_SZ;
+ request_size = round_up(request_size, EROFS_SLOTSIZE);
+ }
+
+ indexes = malloc(sizeof(*indexes) + request_size);
+ if (!indexes) {
+ erofs_err("Failed to alloc memory for index info structure");
+ exit(EXIT_FAILURE);
+ }
+
+ erofs_meta_node_init(&indexes->i_meta_node, EROFS_META_INDEX);
+ indexes->i_1st_idx = first_idx;
+ indexes->i_nidxs = nidxs;
+ memset(indexes->i_idxs, 0, request_size);
+ list_add_tail(&indexes->i_node, &inode->i_compr_idxs_list);
+
+ node = &indexes->i_meta_node;
+ node->m_blk = blk;
+ node->m_type = EROFS_META_INDEX;
+
+ node->m_slot = blk->bb_free_slot;
+ blk->bb_free_slot += request_size / EROFS_SLOTSIZE;
+
+ node->m_len = request_size;
+ list_add_tail(&node->m_node, &blk->bb_metadata_list);
+
+ noninline_nidxs -= nidxs;
+ first_idx += nidxs;
+
+ next = list_next_entry(blk, bb_global_node);
+ erofs_put_block_buffer(blk);
+ blk = next;
+ }
+}
+
+struct erofs_node_info *mkfs_prepare_root_inode(char *root)
+{
+ if (!root)
+ return NULL;
+ return erofs_init_inode(root);
+}
+
+void mkfs_relocate_sub_inodes(struct erofs_node_info *inode)
+{
+ u32 blkaddr;
+ u32 nblocks;
+ u32 unaligned;
+ struct erofs_node_info *d = inode;
+
+ switch (d->i_type) {
+ case EROFS_FT_REG_FILE:
+ case EROFS_FT_DIR:
+ case EROFS_FT_SYMLINK:
+ unaligned = d->i_size % EROFS_BLKSIZE;
+ nblocks = d->i_size / EROFS_BLKSIZE;
+
+ if (unaligned > erofs_calc_inline_data_size(d) ||
+ (unaligned == 0 && nblocks != 0)) {
+ d->i_dmode = EROFS_INODE_LAYOUT_PLAIN;
+ mkfs_rank_inode(d);
+
+ if (unaligned != 0)
+ nblocks++;
+ blkaddr = erofs_alloc_blocks(nblocks);
+ if (!blkaddr)
+ exit(EXIT_FAILURE);
+
+ d->i_blkaddr = blkaddr;
+ } else {
+ d->i_dmode = EROFS_INODE_LAYOUT_INLINE;
+ d->i_inline_datalen = unaligned;
+ mkfs_rank_inode(d);
+
+ if (nblocks > 0) {
+ blkaddr = erofs_alloc_blocks(nblocks);
+ if (!blkaddr)
+ exit(EXIT_FAILURE);
+
+ d->i_blkaddr = blkaddr;
+ } else {
+ d->i_blkaddr = 0;
+ }
+ }
+ break;
+ case EROFS_FT_BLKDEV:
+ case EROFS_FT_CHRDEV:
+ case EROFS_FT_FIFO:
+ case EROFS_FT_SOCK:
+ mkfs_rank_inode(d);
+ break;
+
+ default:
+ erofs_err("inode[%s] file_type error =%d",
+ d->i_fullpath, d->i_type);
+ exit(EXIT_FAILURE);
+ }
+
+ list_for_each_entry(d, &inode->i_subdir_head, i_list) {
+ mkfs_relocate_sub_inodes(d);
+ }
+}
+
+static u32 write_dirents(char *buf, u32 sum,
+ struct list_head *start,
+ struct list_head *end)
+{
+ char *pbuf = buf;
+ u32 size = 0;
+ u32 base_nameoff = 0;
+ struct erofs_dirent dirent;
+ struct list_head *start_tmp = NULL;
+
+
+ base_nameoff = sum * EROFS_DIRENT_SIZE;
+ start_tmp = start;
+ while (start_tmp != end) {
+ struct erofs_node_info *d = container_of(start_tmp,
+ struct erofs_node_info,
+ i_list);
+ u32 name_len = strlen(d->i_name);
+
+ d->i_nameoff = base_nameoff;
+ memcpy(pbuf + base_nameoff, d->i_name, name_len);
+ base_nameoff += name_len;
+ start_tmp = start_tmp->next;
+ }
+
+ start_tmp = start;
+ while (start_tmp != end) {
+ struct erofs_node_info *d = container_of(start_tmp,
+ struct erofs_node_info,
+ i_list);
+ memset(&dirent, 0, EROFS_DIRENT_SIZE);
+
+ dirent.nid = cpu_to_le64(mkfs_addr_to_nid(d->i_base_addr));
+ dirent.file_type = d->i_type;
+ dirent.nameoff = cpu_to_le16(d->i_nameoff);
+ memcpy(pbuf + size, &dirent, EROFS_DIRENT_SIZE);
+ size += EROFS_DIRENT_SIZE;
+ start_tmp = start_tmp->next;
+ }
+ assert(base_nameoff <= EROFS_BLKSIZE);
+
+ return base_nameoff;
+}
+static void mkfs_write_inode_dir(struct erofs_node_info *inode)
+{
+ struct list_head *pos;
+ struct list_head *start;
+ u32 sum = 0;
+ u32 blk_cnt = 0;
+ u32 dentrys_size = 0;
+ char *pbuf = NULL;
+ int ret = 0;
+
+
+ /* dentrys were at inline area */
+ if (inode->i_dmode == EROFS_INODE_LAYOUT_INLINE) {
+ start = (&inode->i_subdir_head)->next;
+ /* dentry begin from the next block offset to inode,
+ * so page_num should be 1
+ */
+ pbuf = calloc(EROFS_BLKSIZE, 1);
+
+ if (!pbuf) {
+ erofs_err("calloc inode[%s] error[%s]",
+ inode->i_fullpath, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ list_for_each(pos, &inode->i_subdir_head) {
+ struct erofs_node_info *d;
+ u32 len;
+
+ d = container_of(pos, struct erofs_node_info, i_list);
+ len = strlen(d->i_name);
+
+ if (dentrys_size + EROFS_DIRENT_SIZE + len >
+ EROFS_BLKSIZE) {
+ u32 addr = inode->i_blkaddr + blk_cnt;
+
+ write_dirents(pbuf, sum, start, pos);
+ ret = dev_write(pbuf, BLKNO_TO_ADDR(addr),
+ EROFS_BLKSIZE);
+ if (ret < 0) {
+ erofs_err("dev_write inode[%s] error[%s]",
+ inode->i_fullpath,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ blk_cnt += 1;
+ sum = 0;
+ dentrys_size = 0;
+ start = pos;
+ }
+
+ dentrys_size += EROFS_DIRENT_SIZE + len;
+ sum += 1;
+
+ }
+
+ /* write last page names */
+ if (start != pos) {
+ s32 len = write_dirents(pbuf, sum, start, pos);
+
+ inode->i_inline_data = pbuf;
+ inode->i_inline_datalen = len;
+ }
+
+ } else if (inode->i_dmode == EROFS_INODE_LAYOUT_PLAIN) {
+ start = (&inode->i_subdir_head)->next;
+ /* dentry begin from the next block offset to inode,
+ * so page_num should be 1
+ */
+ pbuf = calloc(EROFS_BLKSIZE, 1);
+ if (!pbuf) {
+ perror("calloc");
+ exit(EXIT_FAILURE);
+ }
+
+ list_for_each(pos, &inode->i_subdir_head) {
+ struct erofs_node_info *d;
+ u32 len;
+
+ d = container_of(pos, struct erofs_node_info, i_list);
+ len = strlen(d->i_name);
+ if (dentrys_size + EROFS_DIRENT_SIZE + len >
+ EROFS_BLKSIZE) {
+ u32 addr = inode->i_blkaddr + blk_cnt;
+
+ write_dirents(pbuf, sum, start, pos);
+ dev_write(pbuf, BLKNO_TO_ADDR(addr),
+ EROFS_BLKSIZE);
+ if (ret < 0) {
+ erofs_err("dev_write inode[%s] error[%s]",
+ inode->i_fullpath,
+ strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ blk_cnt += 1;
+ sum = 0;
+ dentrys_size = 0;
+ start = pos;
+ }
+
+ dentrys_size += EROFS_DIRENT_SIZE + len;
+ sum += 1;
+
+ }
+
+ /* write last page names */
+ if (start != pos) {
+ u32 addr = inode->i_blkaddr + blk_cnt;
+
+ write_dirents(pbuf, sum, start, pos);
+ ret = dev_write(pbuf, BLKNO_TO_ADDR(addr),
+ EROFS_BLKSIZE);
+ if (ret < 0) {
+ erofs_err("dev_write inode[%s] error[%s]",
+ inode->i_fullpath, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ free(pbuf);
+
+ } else {
+ erofs_err("inode->i_dmode[%u]mode 1 is not support right now",
+ inode->i_dmode);
+ exit(EXIT_FAILURE);
+ }
+}
+
+
+static void mkfs_write_inode_regfile(struct erofs_node_info *inode)
+{
+ char *pbuf = NULL;
+ int ret = 0;
+ u32 i = 0;
+ int fd = 0;
+ u32 nblocks = 0;
+ int unaligned = 0;
+ u32 page_cnt = inode->i_size / EROFS_BLKSIZE;
+ char *filepath = inode->i_fullpath;
+
+ fd = open(filepath, O_RDONLY);
+ if (fd < 0) {
+ perror("fopen() failed");
+ erofs_err("current path=%s", getcwd(NULL, 0));
+ exit(EXIT_FAILURE);
+ }
+
+ switch (inode->i_dmode) {
+ case EROFS_INODE_LAYOUT_PLAIN:
+ if ((inode->i_size % EROFS_BLKSIZE) != 0)
+ page_cnt += 1;
+
+ pbuf = calloc(EROFS_BLKSIZE, 1);
+ if (!pbuf) {
+ erofs_err("calloc inode[%s] error[%s]",
+ inode->i_fullpath, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ for (i = 0; i < page_cnt; i++) {
+ u32 addr;
+
+ ret = read(fd, pbuf, EROFS_BLKSIZE);
+ if (ret < 0) {
+ erofs_err("read inode[%s] error[%s]",
+ filepath, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ addr = inode->i_blkaddr + i;
+
+ ret = dev_write(pbuf, BLKNO_TO_ADDR(addr),
+ EROFS_BLKSIZE);
+ if (ret < 0) {
+ erofs_err("dev_write inode[%s] ret[%d]",
+ filepath, ret);
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ free(pbuf);
+ break;
+
+ case EROFS_INODE_LAYOUT_COMPRESSION:
+ /* it isn't supported right now */
+ assert(0);
+
+ case EROFS_INODE_LAYOUT_INLINE:
+ if (inode->i_size == 0)
+ break;
+
+ nblocks = inode->i_size / EROFS_BLKSIZE;
+ unaligned = inode->i_size % EROFS_BLKSIZE;
+ if (nblocks > 0) {
+ assert(inode->i_blkaddr != 0);
+
+ pbuf = calloc(EROFS_BLKSIZE, 1);
+ if (!pbuf) {
+ erofs_err("calloc inode[%s] error[%s]",
+ inode->i_fullpath, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ for (i = 0; i < nblocks; i++) {
+ u32 addr;
+
+ ret = read(fd, pbuf, EROFS_BLKSIZE);
+ if (ret < 0) {
+ erofs_err("read inode[%s] error[%s]",
+ filepath, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+ addr = inode->i_blkaddr + i;
+
+ ret = dev_write(pbuf, BLKNO_TO_ADDR(addr),
+ EROFS_BLKSIZE);
+ if (ret < 0) {
+ erofs_err("dev_write inode[%s] ret[%d]",
+ filepath, ret);
+ exit(EXIT_FAILURE);
+ }
+ }
+
+ free(pbuf);
+ }
+
+ if (unaligned > 0) {
+ s32 len;
+
+ inode->i_inline_data = calloc(EROFS_BLKSIZE, 1);
+ if (!inode->i_inline_data) {
+ erofs_err("calloc inode[%s] error[%s]",
+ filepath, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ (void)lseek(fd, nblocks * EROFS_BLKSIZE, SEEK_SET);
+
+ len = read(fd, inode->i_inline_data, unaligned);
+ if (len < 0) {
+ erofs_err("read inode[%s] error[%s]",
+ filepath, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+ inode->i_inline_datalen = len;
+ }
+ break;
+
+ default:
+ erofs_err("Inode[%s] mode error [%d]",
+ filepath, inode->i_dmode);
+ exit(EXIT_FAILURE);
+ break;
+ }
+
+ close(fd);
+}
+
+static void mkfs_write_inode_symfile(struct erofs_node_info *inode)
+{
+ char *pbuf = NULL;
+ int ret = 0;
+
+ switch (inode->i_dmode) {
+ case EROFS_INODE_LAYOUT_PLAIN:
+ pbuf = calloc(EROFS_BLKSIZE, 1);
+ if (!pbuf) {
+ erofs_err("calloc inode[%s] error[%s]",
+ inode->i_fullpath, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ ret = readlink(inode->i_fullpath, pbuf, inode->i_size);
+ if (ret < 0) {
+ erofs_err("readlink inode[%s] error[%s]",
+ inode->i_fullpath, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+
+ ret = dev_write(pbuf,
+ BLKNO_TO_ADDR(inode->i_blkaddr),
+ EROFS_BLKSIZE);
+ if (ret < 0) {
+ erofs_err("dev_write inode[%s] error[%s]",
+ inode->i_fullpath, strerror(errno));
+ exit(EXIT_FAILURE);
+ }
+ free(pbuf);
+ break;
+
+ case EROFS_INODE_LAYOUT_COMPRESSION:
+ break;
+
+ case EROFS_INODE_LAYOUT_INLINE:
+ if (inode->i_size == 0)
+ break;
+
+ inode->i_inline_data = calloc(EROFS_BLKSIZE, 1);
+ if (!inode->i_inline_data) {
+ perror("calloc");
+ erofs_info("data_buf is NULL");
+ exit(EXIT_FAILURE);
+ }
+
+ inode->i_inline_datalen = readlink(inode->i_fullpath,
+ inode->i_inline_data,
+ inode->i_size);
+ if (inode->i_inline_datalen < 0) {
+ perror("readlink");
+ exit(EXIT_FAILURE);
+ }
+
+ break;
+
+ default:
+ erofs_err("Inode mode error [%d]", inode->i_dmode);
+ exit(EXIT_FAILURE);
+ break;
+ }
+
+}
+
+void mkfs_do_write_inodes_data(struct erofs_node_info *inode)
+{
+ struct list_head *pos;
+
+ switch (inode->i_type) {
+ case EROFS_FT_DIR:
+ mkfs_write_inode_dir(inode);
+ break;
+
+ case EROFS_FT_REG_FILE:
+ mkfs_write_inode_regfile(inode);
+ break;
+
+ case EROFS_FT_SYMLINK:
+ mkfs_write_inode_symfile(inode);
+ break;
+
+ default:
+ /* special devices file eg. chr block pipe sock */
+ break;
+ }
+
+ if (!list_empty(&inode->i_subdir_head)) {
+ list_for_each(pos, &inode->i_subdir_head) {
+ struct erofs_node_info *d;
+
+ d = container_of(pos, struct erofs_node_info, i_list);
+ mkfs_do_write_inodes_data(d);
+ }
+ }
+
+}
+
+static int erofs_do_write_inode_buffer(struct erofs_node_info *inode,
+ char *buf)
+{
+ struct erofs_inode_v1 *v1;
+ struct erofs_inode_v2 *v2;
+
+ if (inode->i_iver == EROFS_INODE_LAYOUT_V1) {
+ v1 = (struct erofs_inode_v1 *)buf;
+ v1->i_advise = cpu_to_le16(inode->i_iver|(inode->i_dmode<<1));
+ v1->i_xattr_icount = cpu_to_le16(inode->i_xattr_scnt);
+ v1->i_mode = cpu_to_le16(inode->i_mode);
+ v1->i_nlink = cpu_to_le16((u16)inode->i_nlink);
+ v1->i_size = cpu_to_le32((u32)inode->i_size);
+ v1->i_ino = cpu_to_le32(inode->i_ino);
+ v1->i_uid = cpu_to_le16((u16)inode->i_uid);
+ v1->i_gid = cpu_to_le16((u16)inode->i_gid);
+
+ switch (inode->i_type) {
+ case EROFS_FT_CHRDEV:
+ case EROFS_FT_BLKDEV:
+ case EROFS_FT_FIFO:
+ case EROFS_FT_SOCK:
+ v1->i_u.rdev = cpu_to_le32(inode->i_rdev);
+ break;
+
+ default:
+ if (inode->i_dmode == EROFS_INODE_LAYOUT_COMPRESSION)
+ v1->i_u.compressed_blocks =
+ cpu_to_le32(inode->i_blocks);
+ else
+ v1->i_u.raw_blkaddr =
+ cpu_to_le32(inode->i_blkaddr);
+
+ break;
+ }
+
+ v1->i_checksum = 0;
+ return EROFS_INODE_V1_SIZE;
+ } else {
+ v2 = (struct erofs_inode_v2 *)buf;
+ v2->i_advise = cpu_to_le16(inode->i_iver|(inode->i_dmode<<1));
+ v2->i_xattr_icount = cpu_to_le16(inode->i_xattr_scnt);
+ v2->i_mode = cpu_to_le16(inode->i_mode);
+ v2->i_size = cpu_to_le64(inode->i_size);
+ v2->i_u.raw_blkaddr = cpu_to_le32(inode->i_blkaddr);
+ v2->i_ino = cpu_to_le32(inode->i_ino);
+ v2->i_uid = cpu_to_le32(inode->i_uid);
+ v2->i_gid = cpu_to_le32(inode->i_gid);
+ v2->i_ctime = cpu_to_le64(inode->i_ctime);
+ v2->i_ctime_nsec = cpu_to_le32(inode->i_ctime_nsec);
+ v2->i_nlink = cpu_to_le32(inode->i_nlink);
+
+ switch (inode->i_type) {
+ case EROFS_FT_CHRDEV:
+ case EROFS_FT_BLKDEV:
+ case EROFS_FT_FIFO:
+ case EROFS_FT_SOCK:
+ v2->i_u.rdev = cpu_to_le32(inode->i_rdev);
+ break;
+
+ default:
+ if (inode->i_dmode == EROFS_INODE_LAYOUT_COMPRESSION)
+ v2->i_u.compressed_blocks =
+ cpu_to_le32(inode->i_blocks);
+ else
+ v2->i_u.raw_blkaddr =
+ cpu_to_le32(inode->i_blkaddr);
+
+ break;
+ }
+
+ v2->i_checksum = 0;
+ return EROFS_INODE_V2_SIZE;
+ }
+}
+
+int erofs_write_inode_buffer(struct erofs_node_info *inode, char *buf)
+{
+ char *pbuf = buf;
+ int count = 0;
+
+ count += erofs_do_write_inode_buffer(inode, pbuf + count);
+
+ switch (inode->i_dmode) {
+ /* Compress File */
+ case EROFS_INODE_LAYOUT_COMPRESSION:
+ /* Inline softlink or dir or file */
+ case EROFS_INODE_LAYOUT_INLINE:
+ count = round_up(count, inode->i_inline_align_size);
+ if (inode->i_size > 0) {
+ memcpy(pbuf + count,
+ inode->i_inline_data,
+ inode->i_inline_datalen);
+ count += inode->i_inline_datalen;
+ }
+ break;
+
+ default:
+ break;
+ }
+
+ count = SLOT_ALIGN(count);
+ return count;
+}
+
+int erofs_write_index_buffer(struct erofs_index_info *index, char *buf)
+{
+ int count;
+
+ assert(index->i_nidxs);
+
+ count = index->i_nidxs * EROFS_DECOMPR_IDX_SZ;
+ memcpy(buf, index->i_idxs, count);
+
+ return count;
+}
+
+u64 mkfs_addr_to_nid(u64 addr)
+{
+ if (!IS_SLOT_ALIGN(addr)) {
+ erofs_err("SLOT NOT ALIGN: addr=0x%08" PRIX64 "", addr);
+ exit(EXIT_FAILURE);
+ }
+ return (u64)((addr - (u64)sb->meta_blkaddr * EROFS_BLKSIZE) / 32);
+}
diff --git a/mkfs_inode.h b/mkfs_inode.h
new file mode 100644
index 0000000..5758c73
--- /dev/null
+++ b/mkfs_inode.h
@@ -0,0 +1,84 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_inode.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef CONFIG_EROFS_MKFS_INODE_H
+#define CONFIG_EROFS_MKFS_INODE_H
+
+#include "list_head.h"
+#include "erofs_cache.h"
+#include "mkfs_file.h"
+
+#define MAX_PATH 4096
+#define MAX_NAME 256
+
+
+struct erofs_index_info {
+ /* Link to its block buffer */
+ struct erofs_meta_node i_meta_node;
+
+ /* Link to its inode */
+ struct list_head i_node;
+
+ u64 i_1st_idx;
+ s32 i_nidxs;
+ struct z_erofs_vle_decompressed_index i_idxs[0];
+};
+
+struct erofs_node_info {
+ struct erofs_meta_node i_meta_node;
+
+ /* Original member */
+ struct list_head i_list;
+ struct list_head i_subdir_head; /* sub dirs or files */
+ struct list_head i_xattr_head;
+
+ u64 i_base_addr; /* base address of a inode */
+ char i_name[MAX_NAME]; /* the name of current inode */
+ char i_fullpath[MAX_PATH + 1];
+ u16 i_nameoff;
+ u16 i_iver; /* Inode Version */
+ u16 i_dmode; /* Data mode */
+ u16 i_xattr_scnt; /* Inline xattr space count */
+ u16 i_shared_count;
+ u16 i_mode;
+ u8 i_type; /* Inode type: File, Dir...*/
+ u64 i_size;
+ union {
+ u32 i_blkaddr;
+ u32 i_blocks;
+ u32 i_rdev;
+ };
+ u32 i_ino;
+ u32 i_uid;
+ u32 i_gid;
+ u64 i_ctime;
+ u32 i_ctime_nsec;
+ u32 i_nlink;
+
+ /* If compress file, we use it store index info */
+ char *i_inline_data;
+ s32 i_inline_datalen;
+ s32 i_inline_align_size;
+
+ struct erofs_compr_info i_compressor;
+ struct erofs_compr_ctx i_compr_ctx;
+ u64 i_compr_nidxs;
+ u32 i_compr_inlined_nidxs;
+ struct list_head i_compr_idxs_list;
+ struct erofs_index_info *i_compr_cur_index_info;
+};
+
+struct erofs_node_info *mkfs_prepare_root_inode(char *root);
+void mkfs_relocate_sub_inodes(struct erofs_node_info *droot);
+void mkfs_do_write_inodes_data(struct erofs_node_info *droot);
+u64 mkfs_addr_to_nid(u64 addr);
+int erofs_write_inode_buffer(struct erofs_node_info *inode, char *buf);
+int erofs_write_index_buffer(struct erofs_index_info *index, char *buf);
+u8 erofs_check_disk_inode_version(struct erofs_node_info *inode);
+
+#endif
diff --git a/mkfs_main.c b/mkfs_main.c
index ae92656..6e1423f 100644
--- a/mkfs_main.c
+++ b/mkfs_main.c
@@ -17,6 +17,7 @@
#include <libgen.h>
#include "mkfs_erofs.h"
#include "erofs_io.h"
+#include "mkfs_inode.h"
#define pr_fmt(fmt) "MKFS: "FUNC_LINE_FMT fmt"\n"
#include "erofs_debug.h"
@@ -129,6 +130,8 @@ void mkfs_update_erofs_header(u64 root_addr)
}
sb->meta_blkaddr = cpu_to_le32(ADDR_TO_BLKNO(size));
+ sb->blocks = cpu_to_le32(erofs_get_total_blocks());
+ sb->root_nid = cpu_to_le16(mkfs_addr_to_nid(root_addr));
sb_buf = calloc(size, 1);
if (!sb_buf) {
@@ -154,12 +157,37 @@ int main(int argc, char **argv)
mkfs_init_configure();
mkfs_parse_options_cfg(argc, argv);
+ proot_node = mkfs_prepare_root_inode(erofs_cfg.c_src_path);
+ if (!proot_node)
+ goto exit;
+ err = erofs_create_files_list(proot_node);
+ if (err)
+ goto exit;
+
+ err = erofs_cache_init(ADDR_TO_BLKNO(BLK_ALIGN(EROFS_SUPER_END)));
+ if (err)
+ goto exit;
+
+ mkfs_relocate_sub_inodes(proot_node);
+
+ mkfs_do_write_inodes_data(proot_node);
+
+ err = erofs_flush_all_blocks();
+ if (err)
+ goto exit;
+
mkfs_update_erofs_header(proot_node->i_base_addr);
dev_close();
mkfs_free_config();
erofs_info("done");
+exit:
+
+ if (err)
+ erofs_err("\tError: Could not format the device!!!\n");
+
+ erofs_cache_deinit();
return err;
}
--
2.17.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [PREVIEW] [RFC PATCH v2 4/5] erofs-mkfs: add compression support
2018-11-20 14:32 [PREVIEW] [RFC PATCH v2 0/5] erofs-mkfs: mkfs.erofs source code first preview Li Guifu
` (2 preceding siblings ...)
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 3/5] erofs-mkfs: support to build a image from the specific root directory Li Guifu
@ 2018-11-20 14:32 ` Li Guifu
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 5/5] erofs-mkfs: add miscellaneous files Li Guifu
4 siblings, 0 replies; 6+ messages in thread
From: Li Guifu @ 2018-11-20 14:32 UTC (permalink / raw)
This patch introduces a compression framework, so it can support
different compression algorithm in principle, currently lz4hc is
added right now due to the kernel implementation.
Signed-off-by: Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Fang Wei <fangwei1 at huawei.com>
---
erofs_compressor.c | 144 ++++++++++
erofs_compressor.h | 49 ++++
erofs_lz4hc.c | 52 ++++
erofs_lz4hc.h | 22 ++
mkfs_file.c | 698 +++++++++++++++++++++++++++++++++++++++++++++
mkfs_inode.c | 61 +++-
mkfs_main.c | 4 +
7 files changed, 1028 insertions(+), 2 deletions(-)
create mode 100644 erofs_compressor.c
create mode 100644 erofs_compressor.h
create mode 100644 erofs_lz4hc.c
create mode 100644 erofs_lz4hc.h
diff --git a/erofs_compressor.c b/erofs_compressor.c
new file mode 100644
index 0000000..139ae53
--- /dev/null
+++ b/erofs_compressor.c
@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs_compressor.c
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Miao Xie <miaoxie at huawei.com>
+ */
+#include <string.h>
+#include <assert.h>
+
+#include "erofs_error.h"
+#include "erofs_compressor.h"
+#include "erofs_lz4hc.h"
+#include "erofs_debug.h"
+#include "mkfs_erofs.h"
+
+static struct erofs_compr_alg erofs_compr_desc[EROFS_COMPR_ALG_MAX] = {
+ [EROFS_COMPR_NONE] = {
+ .ca_name = "none",
+ .ca_idx = EROFS_COMPR_NONE,
+ .ca_max_lvl = 0,
+ .ca_min_lvl = 0,
+ .ca_def_lvl = 0,
+ .ca_compress = NULL,
+ .ca_init = NULL,
+ .ca_deinit = NULL,
+ },
+ [EROFS_COMPR_LZ4HC] = {
+ .ca_name = "lz4hc",
+ .ca_idx = EROFS_COMPR_LZ4HC,
+ .ca_max_lvl = LZ4HC_CLEVEL_MAX,
+ .ca_min_lvl = LZ4HC_CLEVEL_MIN,
+ .ca_def_lvl = EROFS_COMPR_LZ4HC_DEF_LVL,
+ .ca_compress = erofs_lz4hc_compress,
+ .ca_init = erofs_lz4hc_init,
+ .ca_deinit = erofs_lz4hc_deinit,
+ },
+};
+
+void erofs_compress_alg_init(const char *name)
+{
+ int level;
+ struct erofs_compr_alg *alg;
+
+ if (!name) {
+ erofs_err("compress alg name is NULL !!!");
+ exit(EXIT_FAILURE);
+ }
+
+ /* name: lz4hc or none */
+ alg = erofs_get_compress_alg(name);
+ if (!alg) {
+ erofs_err("can found alg[%s]", name);
+ exit(EXIT_FAILURE);
+ }
+ erofs_cfg.c_compr_alg = alg;
+ erofs_cfg.c_compr_maxsz = BLK_ALIGN(EROFS_CONFIG_COMPR_MAX_SZ);
+
+ level = erofs_adjust_compress_level(alg, EROFS_COMPR_LZ4HC_DEF_LVL);
+ erofs_cfg.c_compr_lvl = level;
+ erofs_cfg.c_compr_boundary = EROFS_CONFIG_COMPR_DEF_BOUNDARY;
+ erofs_cfg.c_compr_ratio_limit = EROFS_CONFIG_COMPR_RATIO_MAX_LIMIT;
+
+}
+struct erofs_compr_alg *erofs_get_compress_alg(const char *name)
+{
+ int i;
+
+ for (i = EROFS_COMPR_NONE; i < EROFS_COMPR_ALG_MAX; i++) {
+ if (strcmp(name, erofs_compr_desc[i].ca_name) == 0)
+ return &erofs_compr_desc[i];
+ }
+
+ return NULL;
+}
+
+int erofs_adjust_compress_level(struct erofs_compr_alg *alg, int lvl)
+{
+ if (!alg || alg->ca_idx == EROFS_COMPR_NONE)
+ return 0;
+
+ if (lvl > alg->ca_max_lvl) {
+ erofs_err("Compress level(%d) is greater than max level(%d), adjust it to default level(%d).\n",
+ lvl, alg->ca_max_lvl, EROFS_COMPR_LZ4HC_DEF_LVL);
+ return alg->ca_def_lvl;
+ }
+
+ if (lvl < alg->ca_min_lvl) {
+ erofs_err("Compress level(%d) is less than min level(%d), adjust it to default level(%d).\n",
+ lvl, alg->ca_min_lvl, EROFS_COMPR_LZ4HC_DEF_LVL);
+ return alg->ca_def_lvl;
+ }
+
+ return lvl;
+}
+
+void *erofs_compress_init(struct erofs_compr_alg *alg)
+{
+ void *ctx;
+
+ if (!alg->ca_init)
+ return NULL;
+
+ ctx = alg->ca_init();
+
+ return ctx;
+}
+
+void erofs_compress_deinit(struct erofs_compr_alg *alg, void *cctx)
+{
+ if (!alg->ca_deinit)
+ return;
+
+ alg->ca_deinit(cctx);
+}
+
+int64_t erofs_compress_onctx(struct erofs_compr_alg *alg, void *ctx,
+ char *in, size_t insz,
+ char *out, size_t outsz, size_t *srcsz, int lvl)
+{
+ assert(alg->ca_compress);
+
+ return alg->ca_compress(in, insz, out, outsz, srcsz, lvl, ctx);
+}
+
+int64_t erofs_compress(struct erofs_compr_alg *alg, char *in, size_t insz,
+ char *out, size_t outsz, size_t *srcsz, int lvl)
+{
+ void *ctx = NULL;
+ int64_t ret;
+
+ if (alg->ca_init) {
+ ctx = alg->ca_init();
+ if (IS_ERR(ctx))
+ return EROFS_COMPRESS_ERROR;
+ }
+
+ ret = alg->ca_compress(in, insz, out, outsz, srcsz, lvl, ctx);
+ if (alg->ca_deinit)
+ alg->ca_deinit(ctx);
+
+ return ret;
+}
diff --git a/erofs_compressor.h b/erofs_compressor.h
new file mode 100644
index 0000000..f2d8b09
--- /dev/null
+++ b/erofs_compressor.h
@@ -0,0 +1,49 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_compressor.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_COMPRESSOR_H__
+#define __EROFS_COMPRESSOR_H__
+
+#include <stdint.h>
+
+#define EROFS_COMPRESS_ERROR (-1LL)
+
+enum erofs_compr_algs {
+ EROFS_COMPR_NONE,
+ EROFS_COMPR_LZ4HC,
+ EROFS_COMPR_ALG_MAX,
+};
+
+typedef int64_t (*compress_func)(char *in, size_t insize, char *out,
+ size_t outsize, size_t *insizeptr, int level,
+ void*);
+typedef void* (*init_func)();
+typedef void (*deinit_func)(void *cctx);
+
+struct erofs_compr_alg {
+ char *ca_name;
+ int ca_idx;
+ int ca_max_lvl;
+ int ca_min_lvl;
+ int ca_def_lvl;
+ compress_func ca_compress;
+ init_func ca_init;
+ deinit_func ca_deinit;
+};
+
+void erofs_compress_alg_init(const char *name);
+struct erofs_compr_alg *erofs_get_compress_alg(const char *name);
+int erofs_adjust_compress_level(struct erofs_compr_alg *alg, int lvl);
+void *erofs_compress_init(struct erofs_compr_alg *alg);
+void erofs_compress_deinit(struct erofs_compr_alg *alg, void *cctx);
+int64_t erofs_compress_onctx(struct erofs_compr_alg *alg, void *ctx,
+ char *in, size_t insz,
+ char *out, size_t outsz, size_t *srcsz, int lvl);
+int64_t erofs_compress(struct erofs_compr_alg *alg, char *in, size_t insz,
+ char *out, size_t outsz, size_t *srcsz, int lvl);
+#endif
diff --git a/erofs_lz4hc.c b/erofs_lz4hc.c
new file mode 100644
index 0000000..1d155b7
--- /dev/null
+++ b/erofs_lz4hc.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs_lz4hc.c
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#include <errno.h>
+#define LZ4_HC_STATIC_LINKING_ONLY (1)
+#include <lz4hc.h>
+
+#include "erofs_error.h"
+#include "erofs_lz4hc.h"
+#include "erofs_compressor.h"
+#include "erofs_debug.h"
+
+void *erofs_lz4hc_init(void)
+{
+ LZ4_streamHC_t *ctx;
+
+ ctx = LZ4_createStreamHC();
+ if (!ctx) {
+ erofs_err("Cannot allocate LZ4HC context");
+ return ERR_PTR(-ENOMEM);
+ }
+
+ return (void *)ctx;
+}
+
+void erofs_lz4hc_deinit(void *ctx)
+{
+ if (!ctx)
+ return;
+
+ LZ4_freeStreamHC((LZ4_streamHC_t *)ctx);
+}
+
+int64_t erofs_lz4hc_compress(char *in, size_t insz, char *out, size_t outsz,
+ size_t *inszptr, int level, void *ctx)
+{
+ int count;
+
+ *inszptr = insz;
+ count = LZ4_compress_HC_destSize((LZ4_streamHC_t *)ctx, in, out,
+ (int *)inszptr, outsz, level);
+ if (count <= 0) {
+ erofs_err("Failed to compress data by LZ4HC");
+ return EROFS_COMPRESS_ERROR;
+ }
+ return (int64_t)count;
+}
diff --git a/erofs_lz4hc.h b/erofs_lz4hc.h
new file mode 100644
index 0000000..029c4f4
--- /dev/null
+++ b/erofs_lz4hc.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * erofs_lz4hc.h
+ *
+ * Copyright (C) 2018 HUAWEI, Inc.
+ * http://www.huawei.com/
+ * Created by Li Guifu <bluce.liguifu at huawei.com>
+ */
+#ifndef __EROFS_LZ4HC_H__
+#define __EROFS_LZ4HC_H__
+
+#include <stdint.h>
+#include <lz4hc.h>
+
+#define EROFS_COMPR_LZ4HC_DEF_LVL (9)
+
+void *erofs_lz4hc_init(void);
+void erofs_lz4hc_deinit(void *ctx);
+int64_t erofs_lz4hc_compress(char *in, size_t insz, char *out,
+ size_t outsz, size_t *inszptr,
+ int level, void*);
+#endif
diff --git a/mkfs_file.c b/mkfs_file.c
index beeb89d..a979c14 100644
--- a/mkfs_file.c
+++ b/mkfs_file.c
@@ -36,6 +36,7 @@
#include "erofs_types.h"
#include "list_head.h"
#include "erofs_cache.h"
+#include "erofs_compressor.h"
#define pr_fmt(fmt) "MKFS-FILE: "FUNC_LINE_FMT fmt"\n"
#include "erofs_debug.h"
@@ -304,3 +305,700 @@ struct erofs_node_info *alloc_erofs_node(void)
init_list_head(&f->i_xattr_head);
return f;
+}
+
+static void
+erofs_compr_idx_host_to_disk(struct erofs_compr_idx *hidx,
+ struct z_erofs_vle_decompressed_index *didx)
+{
+ switch (hidx->di_advise) {
+ case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
+ case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
+ didx->di_advise = cpu_to_le16(hidx->di_advise);
+ didx->di_clusterofs = cpu_to_le16(hidx->di_clusterofs);
+ didx->di_u.blkaddr = cpu_to_le32(hidx->blkaddr);
+ break;
+
+ case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
+ didx->di_advise = cpu_to_le16(hidx->di_advise);
+ didx->di_clusterofs = cpu_to_le16(hidx->di_clusterofs);
+ didx->di_u.delta[0] = cpu_to_le16(hidx->delta[0]);
+ didx->di_u.delta[1] = cpu_to_le16(hidx->delta[1]);
+ break;
+
+ default:
+ assert(0);
+ break;
+ }
+}
+
+static int erofs_compress_inline_file_data(struct erofs_compr_info *cinfo,
+ struct erofs_compr_ctx *ctx)
+{
+ int64_t compr_count;
+ size_t comprsz = 0;
+
+ assert(ctx->cc_srclen <= EROFS_BLKSIZE);
+ assert(ctx->cc_buflen >= 2 * EROFS_BLKSIZE);
+
+ compr_count = erofs_compress(cinfo->ci_alg,
+ ctx->cc_srcbuf, ctx->cc_srclen,
+ ctx->cc_dstbuf, EROFS_BLKSIZE,
+ &comprsz, cinfo->ci_lvl);
+
+ if (compr_count == 0 || compr_count == EROFS_COMPRESS_ERROR) {
+ erofs_err("Failed to compress data by %s",
+ cinfo->ci_alg->ca_name);
+ return -EIO;
+ }
+
+ assert(comprsz == (size_t)ctx->cc_srclen);
+
+ ctx->cc_dstlen = (int)compr_count;
+ ctx->cc_nidxs = EROFS_COMPR_CTX_INLINED_DATA;
+ return 0;
+}
+
+/* Note: it is not for inline data compress */
+static int
+erofs_compress_noinline_file_data(struct erofs_compr_info *cinfo,
+ struct erofs_compr_ctx *ctx)
+{
+ char *in;
+ char *out;
+ size_t insz;
+ size_t outsz;
+ u32 blkaddr;
+ size_t comprsz;
+ int64_t compr_count;
+ long long pos;
+ int start;
+ int end;
+ int i;
+ int advise;
+ int clusterofs;
+ int delta;
+ int cross;
+ int nidxs;
+ struct erofs_compr_idx *idx;
+
+ in = ctx->cc_srcbuf;
+ insz = ctx->cc_srclen;
+ out = ctx->cc_dstbuf;
+ outsz = EROFS_BLKSIZE;
+ blkaddr = 0;
+ pos = ctx->cc_pos;
+ nidxs = 0;
+
+ assert(pos % EROFS_BLKSIZE == 0);
+ assert(insz % EROFS_BLKSIZE == 0);
+
+ while (insz > 0) {
+ advise = Z_EROFS_VLE_CLUSTER_TYPE_MAX;
+ /* Data is less than a block, don't compress */
+ if (insz <= EROFS_BLKSIZE) {
+ advise = Z_EROFS_VLE_CLUSTER_TYPE_PLAIN;
+ comprsz = insz;
+ compr_count = insz;
+ memcpy(out, in, insz);
+ goto update_index;
+ }
+
+ comprsz = 0;
+ compr_count = erofs_compress(cinfo->ci_alg, in, insz,
+ out, outsz,
+ &comprsz, cinfo->ci_lvl);
+
+ if (compr_count == 0 ||
+ compr_count == EROFS_COMPRESS_ERROR) {
+ erofs_err("Failed to compress data by %s",
+ cinfo->ci_alg->ca_name);
+ return -EIO;
+ }
+
+ /* compress ratio is very bad, don't compress */
+ if ((int)comprsz - (int)compr_count <
+ erofs_cfg.c_compr_boundary) {
+ advise = Z_EROFS_VLE_CLUSTER_TYPE_PLAIN;
+
+ if (pos % EROFS_BLKSIZE == 0)
+ comprsz = EROFS_BLKSIZE;
+ else
+ comprsz = (int)(round_up(pos,
+ EROFS_BLKSIZE) - pos);
+
+ compr_count = comprsz;
+ memcpy(out, in, comprsz);
+ goto update_index;
+ }
+
+ if ((pos + comprsz) % EROFS_BLKSIZE <=
+ (unsigned int)erofs_cfg.c_compr_boundary)
+ comprsz -= (int)((pos + comprsz) % EROFS_BLKSIZE);
+
+ assert(comprsz);
+
+update_index:
+ start = (int)((pos - ctx->cc_pos) / EROFS_BLKSIZE);
+ end = (int)((pos + comprsz - ctx->cc_pos) / EROFS_BLKSIZE);
+
+ assert(end > start);
+
+ if ((pos + comprsz) % EROFS_BLKSIZE != 0)
+ cross = end - start + 1;
+ else
+ cross = end - start;
+
+ clusterofs = pos % PAGE_SIZE;
+ delta = 0;
+
+ /*
+ * Here we against the rule that the length of code should
+ * less than 80 bytes, it is because we want to make
+ * the readability of mathematical expression be better.
+ */
+ erofs_dbg("Compress range(Original[%lld - %lld], Index[%d - %d], Aligned[%lld - %lld], Index[%lld - %lld]) Start index %s, end index %s, end pos %s\n",
+ pos, pos + comprsz - 1,
+ start, end,
+ round_down(pos, EROFS_BLKSIZE),
+ round_up(pos + comprsz - 1, EROFS_BLKSIZE) - 1,
+ (round_down(pos, EROFS_BLKSIZE) - ctx->cc_pos) / EROFS_BLKSIZE,
+ (round_up(pos + comprsz - 1, EROFS_BLKSIZE) - ctx->cc_pos) / EROFS_BLKSIZE - 1,
+ start == (int)(round_down(pos, EROFS_BLKSIZE) - ctx->cc_pos) / EROFS_BLKSIZE ? "SAME" : "DIFF",
+ end == (int)(round_up(pos + comprsz - 1, EROFS_BLKSIZE) - ctx->cc_pos) / EROFS_BLKSIZE - 1 ? "SAME" : "DIFF",
+ pos + comprsz - 1 == round_up(pos + comprsz - 1, EROFS_BLKSIZE) - 1 ? "SAME" : "DIFF");
+
+ for (i = start; i < end; i++) {
+ idx = &ctx->cc_idxs[i];
+ if (advise == Z_EROFS_VLE_CLUSTER_TYPE_MAX) {
+ if (delta == 0) {
+ idx->di_advise =
+ Z_EROFS_VLE_CLUSTER_TYPE_HEAD;
+ } else {
+ idx->di_advise =
+ Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD;
+ }
+ } else {
+ idx->di_advise = advise;
+ }
+ idx->di_clusterofs = clusterofs;
+ idx->delta[0] = delta;
+ idx->delta[1] = cross - delta - 1;
+ /* Allocate the blocks later */
+ idx->blkaddr = blkaddr;
+
+ erofs_dbg("Compress Index: advise - %u, clusterofs - %u, delta0 - %u, delta1 - %u, blkaddr - %u",
+ idx->di_advise, clusterofs,
+ delta, cross - delta,
+ blkaddr);
+ delta++;
+ nidxs++;
+ }
+
+ insz -= comprsz;
+ in += comprsz;
+ out += EROFS_BLKSIZE;
+ pos += comprsz;
+ blkaddr++;
+ }
+
+ ctx->cc_dstlen = (int)(out - ctx->cc_dstbuf);
+ ctx->cc_nidxs = nidxs;
+ return 0;
+}
+
+int erofs_write_compress_data(struct erofs_compr_ctx *cctx)
+{
+ u32 nblocks;
+ u32 blkaddr;
+ int ret;
+ int i;
+
+ nblocks = cctx->cc_dstlen / EROFS_BLKSIZE;
+ blkaddr = erofs_alloc_blocks(nblocks);
+
+ if (!blkaddr)
+ return -ENOSPC;
+
+ ret = dev_write(cctx->cc_dstbuf, BLKNO_TO_ADDR(blkaddr),
+ cctx->cc_dstlen);
+
+ if (ret)
+ return -EIO;
+
+ for (i = 0; i < cctx->cc_nidxs; i++)
+ cctx->cc_idxs[i].blkaddr += blkaddr;
+
+ return 0;
+}
+
+int erofs_update_indexes(struct erofs_node_info *inode,
+ struct erofs_compr_ctx *cctx)
+{
+ u64 index = cctx->cc_pos / EROFS_BLKSIZE;
+ struct erofs_index_info *iinfo;
+ struct z_erofs_vle_decompressed_index *didx;
+ int i;
+ int j = 0;
+ int end;
+
+ iinfo = inode->i_compr_cur_index_info;
+
+ /* Find index struct which we want */
+ if (iinfo && index >= iinfo->i_1st_idx)
+ goto search_next_index_info;
+
+ if (index >= inode->i_compr_inlined_nidxs)
+ goto search_from_1st_index_info;
+
+ didx = (void *)(inode->i_inline_data +
+ sizeof(struct erofs_extent_header));
+ end = inode->i_compr_inlined_nidxs;
+
+ for (i = (int)index; i < end && j < cctx->cc_nidxs; i++, j++)
+ erofs_compr_idx_host_to_disk(&cctx->cc_idxs[j], &didx[i]);
+
+ if (j == cctx->cc_nidxs)
+ return 0;
+
+ index = i;
+search_from_1st_index_info:
+ iinfo = list_first_entry(&inode->i_compr_idxs_list,
+ struct erofs_index_info, i_node);
+search_next_index_info:
+ list_for_each_entry_from(iinfo, &inode->i_compr_idxs_list, i_node) {
+ if (index < iinfo->i_1st_idx + iinfo->i_nidxs)
+ break;
+ }
+
+ assert(index >= iinfo->i_1st_idx);
+
+ do {
+ didx = iinfo->i_idxs;
+ i = index - iinfo->i_1st_idx;
+ end = iinfo->i_nidxs;
+
+ for (; i < end && j < cctx->cc_nidxs; i++, j++)
+ erofs_compr_idx_host_to_disk(&cctx->cc_idxs[j],
+ &didx[i]);
+
+ if (j == cctx->cc_nidxs)
+ break;
+
+ index = i + iinfo->i_1st_idx;
+ iinfo = list_next_entry(iinfo, i_node);
+ } while (1);
+
+ inode->i_compr_cur_index_info = iinfo;
+ return 0;
+}
+
+int erofs_compress_file(struct erofs_node_info *inode)
+{
+ int fd = -1;
+ size_t read_count;
+ off64_t pos = 0;
+ u64 isize = inode->i_size;
+ u64 itotal = 0;
+ u64 ototal = 0;
+ u64 nidxs = 0;
+ int ret = 0;
+ struct erofs_compr_ctx *cctx = &inode->i_compr_ctx;
+ struct erofs_compr_info *compressor = &inode->i_compressor;
+ struct erofs_extent_header *header;
+
+ assert(!inode->i_inline_data);
+ assert(inode->i_size > 0);
+
+ inode->i_inline_data = malloc(EROFS_BLKSIZE);
+
+ if (!inode->i_inline_data) {
+ erofs_err("Fail to alloc inline data buffer(%s)",
+ inode->i_name);
+ return -ENOMEM;
+ }
+
+ memset(inode->i_inline_data, 0, EROFS_BLKSIZE);
+
+ /* Init header */
+ header = (struct erofs_extent_header *)inode->i_inline_data;
+ header->eh_checksum = 0;
+
+ /*
+ * We have compressed some data at the head of the file when we check
+ * the compressible, so we should go to the branch, put a assert here
+ * to check LOGICAL BUG in the code.
+ */
+ if (cctx->cc_pos != 0 || cctx->cc_nidxs == 0) {
+ assert(0);
+ return -EIO;
+ }
+
+ /*
+ * Check cctx, write out the compress data and update the metadatae if
+ * we have compressed some data before.
+ */
+ if (cctx->cc_nidxs == EROFS_COMPR_CTX_INLINED_DATA) {
+ /*
+ * TODO: Now we don't support inlined compress data,
+ * we will implement it in the future, add a assert
+ * here to avoid someone making a mistake.
+ *
+ * ? where can we keep the compress data len? i_blocks?
+ */
+ assert(0);
+ erofs_dbg("Size: %d(%"PRIu64") ==> %d, Inline Compress, Compress Ratio %.2lf%%.\n",
+ cctx->cc_srclen, isize, cctx->cc_dstlen,
+ (double)cctx->cc_dstlen * 100 / (double)cctx->cc_srclen);
+ return 0;
+ } else if (cctx->cc_nidxs < 0) {
+ /* There is something wrong with nidxs */
+ assert(0);
+ return -EIO;
+ }
+
+ ret = erofs_write_compress_data(cctx);
+
+ if (ret)
+ return ret;
+
+ ret = erofs_update_indexes(inode, cctx);
+
+ if (ret)
+ return ret;
+
+ itotal = cctx->cc_srclen;
+ ototal = cctx->cc_dstlen;
+ nidxs = cctx->cc_nidxs;
+
+ pos = cctx->cc_pos + cctx->cc_srclen;
+
+ if ((u64)pos >= inode->i_size)
+ goto compress_complete;
+
+ fd = open(inode->i_fullpath, O_RDONLY | O_BINARY);
+
+ if (fd < 0) {
+ erofs_err("Fail to open a file(%s)", inode->i_name);
+ return -ENOENT;
+ }
+
+ pos = lseek64(fd, pos, SEEK_SET);
+
+ if (pos == (off64_t) -1ULL) {
+ ret = -EINVAL;
+ goto close_file;
+ }
+
+ assert(pos != 0);
+
+ while (1) {
+ erofs_reset_compress_context(cctx);
+
+ read_count = read(fd, cctx->cc_srcbuf, cctx->cc_buflen);
+
+ if (read_count == 0) {
+ if (itotal == isize) {
+ /* EOF, go out and complete compression */
+ ret = 0;
+ } else {
+ /*
+ * Read error happened and the operation was
+ * interrupted.
+ */
+ erofs_err("Read file(%s) interrupted at offset - %lld",
+ inode->i_name, (long long)pos);
+ ret = -EIO;
+ }
+
+ break;
+ }
+
+ itotal += read_count;
+
+ if (itotal > isize) {
+ erofs_err("Read overflow(File: %s, Real Size:%llu, Read Size: %llu)",
+ inode->i_name, (long long unsigned)isize,
+ (long long unsigned)itotal);
+ ret = -EIO;
+ break;
+ } else if (itotal == isize) {
+ read_count = round_up(read_count, EROFS_BLKSIZE);
+ } else {
+ if (read_count % EROFS_BLKSIZE != 0) {
+ erofs_err("Read size is not aligned(File: %s, Pos: %"PRIu64", Size: %zd)",
+ inode->i_name, (u64)pos, read_count);
+ ret = -EIO;
+ break;
+ }
+ }
+
+ cctx->cc_pos = pos;
+ cctx->cc_srclen = read_count;
+
+ ret = erofs_compress_noinline_file_data(compressor, cctx);
+
+ if (ret) {
+ erofs_err("Compress file Fail(File: %s, Pos: %"PRIu64", Size: %zd)",
+ inode->i_name, (u64)pos, read_count);
+ ret = -EIO;
+ break;
+ }
+
+ ret = erofs_write_compress_data(cctx);
+
+ if (ret)
+ break;
+
+ ret = erofs_update_indexes(inode, cctx);
+
+ if (ret)
+ break;
+
+ ototal += cctx->cc_dstlen;
+ nidxs += cctx->cc_nidxs;
+ pos += read_count;
+ }
+
+compress_complete:
+
+ if (!ret) {
+ inode->i_blocks = (u32)(ototal / EROFS_BLKSIZE);
+ erofs_dbg("Size: %"PRIu64"(%"PRIu64") ==> %"PRIu64", Indexs %"PRIu64", Compress Ratio %.2lf%%.\n",
+ itotal, isize, ototal, nidxs,
+ (double)ototal * 100 / (double)itotal);
+ }
+
+close_file:
+
+ if (fd >= 0)
+ close(fd);
+
+ return ret;
+}
+
+int erofs_try_compress_file_once(struct erofs_node_info *inode,
+ struct erofs_compr_info *cinfo,
+ struct erofs_compr_ctx *cctx)
+{
+ int fd;
+ size_t read_count;
+ loff_t pos = 0;
+ u64 isize = inode->i_size;
+ int inlined = 0;
+ int ret = 0;
+
+ assert(cinfo->ci_alg);
+ assert(cinfo->ci_alg->ca_idx != EROFS_COMPR_NONE);
+ assert(cctx->cc_buflen > EROFS_BLKSIZE &&
+ cctx->cc_buflen % EROFS_BLKSIZE == 0);
+ assert(cctx->cc_pos == 0);
+ assert(inode->i_size > 0);
+ assert(inode->i_compressor.ci_alg == NULL);
+ assert(inode->i_compr_ctx.cc_srcbuf == NULL);
+
+ fd = open(inode->i_fullpath, O_RDONLY | O_BINARY);
+
+ if (fd < 0) {
+ erofs_err("Fail to open a file(%s)", inode->i_fullpath);
+ return -ENOENT;
+ }
+
+ read_count = read(fd, cctx->cc_srcbuf, cctx->cc_buflen);
+
+ if (read_count == 0) {
+ erofs_err("Read file(%s) interrupted at offset - %"PRIu64"",
+ inode->i_name, (u64)pos);
+ ret = -EIO;
+ goto close_file;
+ }
+
+ if (read_count > isize) {
+ erofs_err("Read overflow(File: %s, Real Size:%"PRIu64", Read Size: %zd)",
+ inode->i_name, (u64)isize, read_count);
+ ret = -EIO;
+ goto close_file;
+ } else if (read_count == isize) {
+ if (isize > EROFS_BLKSIZE)
+ read_count = round_up(read_count, EROFS_BLKSIZE);
+ else
+ inlined = 1;
+ } else {
+ if (read_count % EROFS_BLKSIZE != 0) {
+ erofs_err("Read size is not aligned(File: %s, Pos: %"PRIu64", Size: %zd)",
+ inode->i_name, (u64)pos, read_count);
+ ret = -EIO;
+ goto close_file;
+ }
+ }
+
+ cctx->cc_pos = 0;
+ cctx->cc_srclen = read_count;
+
+ if (inlined)
+ ret = erofs_compress_inline_file_data(cinfo, cctx);
+ else
+ ret = erofs_compress_noinline_file_data(cinfo, cctx);
+
+ if (ret) {
+ erofs_err("Compress file Fail(File: %s, Pos: %"PRIu64", Size: %zd)",
+ inode->i_name, (u64)pos, read_count);
+ ret = -EIO;
+ }
+
+close_file:
+ close(fd);
+ return ret;
+}
+
+static int erofs_get_node_compress_info(struct erofs_node_info *inode,
+ struct erofs_compr_info *cinfo)
+{
+ /* Get specified compress algorithm which is set in the config file */
+ /*
+ * Now we have not implement it, just use the algorithm
+ * set in command line.
+ */
+ (void)inode;
+ cinfo->ci_alg = erofs_cfg.c_compr_alg;
+ cinfo->ci_lvl = erofs_cfg.c_compr_lvl;
+
+ return 0;
+}
+
+void erofs_deinit_compress_context(struct erofs_compr_ctx *ctx)
+{
+ if (ctx->cc_srcbuf)
+ free(ctx->cc_srcbuf);
+
+ if (ctx->cc_dstbuf)
+ free(ctx->cc_dstbuf);
+
+ if (ctx->cc_idxs)
+ free(ctx->cc_idxs);
+
+ memset(ctx, 0, sizeof(struct erofs_compr_ctx));
+}
+
+int erofs_init_compress_context(struct erofs_compr_ctx *ctx)
+{
+ memset(ctx, 0, sizeof(struct erofs_compr_ctx));
+
+ ctx->cc_srcbuf = malloc(erofs_cfg.c_compr_maxsz);
+ ctx->cc_dstbuf = malloc(erofs_cfg.c_compr_maxsz * 2);
+ ctx->cc_idxs = calloc(erofs_cfg.c_compr_maxsz / EROFS_BLKSIZE,
+ sizeof(struct erofs_compr_idx));
+
+ if (!ctx->cc_srcbuf || !ctx->cc_dstbuf || !ctx->cc_idxs) {
+ erofs_deinit_compress_context(ctx);
+ return -ENOMEM;
+ }
+
+ ctx->cc_buflen = erofs_cfg.c_compr_maxsz;
+
+ memset(ctx->cc_srcbuf, 0, ctx->cc_buflen);
+ memset(ctx->cc_dstbuf, 0, ctx->cc_buflen);
+ memset(ctx->cc_idxs, 0,
+ ctx->cc_buflen / EROFS_BLKSIZE * sizeof(struct erofs_compr_idx));
+
+ return 0;
+}
+
+void erofs_reset_compress_context(struct erofs_compr_ctx *ctx)
+{
+ ctx->cc_pos = 0;
+ ctx->cc_srclen = 0;
+ ctx->cc_dstlen = 0;
+ ctx->cc_nidxs = 0;
+ memset(ctx->cc_srcbuf, 0, ctx->cc_buflen);
+ memset(ctx->cc_dstbuf, 0, ctx->cc_buflen);
+ memset(ctx->cc_idxs, 0,
+ ctx->cc_buflen / EROFS_BLKSIZE * sizeof(struct erofs_compr_idx));
+}
+
+int erofs_check_compressible(struct erofs_node_info *inode)
+{
+ struct erofs_compr_info cinfo;
+ struct erofs_compr_ctx ctx;
+ int ratio;
+ int ret;
+
+ if (erofs_cfg.c_compr_alg->ca_idx == EROFS_COMPR_NONE) {
+ /* Compress is disable by the user */
+ return 0;
+ }
+
+ if (inode->i_type != EROFS_FT_REG_FILE)
+ return 0;
+
+ /* check if we can inline data directly */
+ if (inode->i_size <= erofs_calc_inline_data_size(inode))
+ return 0;
+
+ /* check if the user don't want to compress this file */
+ cinfo.ci_alg = NULL;
+ cinfo.ci_lvl = 0;
+
+ ret = erofs_get_node_compress_info(inode, &cinfo);
+
+ if (ret) {
+ erofs_err("Failed to get compress algorithm for %s",
+ inode->i_name);
+ assert(ret < 0);
+ return ret;
+ }
+
+ if (!cinfo.ci_alg || cinfo.ci_alg->ca_idx == EROFS_COMPR_NONE)
+ return 0;
+
+ assert(erofs_cfg.c_compr_maxsz % EROFS_BLKSIZE == 0);
+ ret = erofs_init_compress_context(&ctx);
+
+ if (ret)
+ return ret;
+
+ ret = erofs_try_compress_file_once(inode, &cinfo, &ctx);
+
+ if (ret) {
+ erofs_deinit_compress_context(&ctx);
+ return ret;
+ }
+
+ /* FIXME: Now we don't implement inline compress, so... */
+ if (inode->i_size <= EROFS_BLKSIZE) {
+ /*
+ * TODO: Now we haven't support inline compress data, so
+ * disable compress if
+ * inline_data_size < i_size <= block_size
+ */
+#ifdef CONFIG_EROFS_INLINE_COMPRESS_DATA
+ if (ctx.dstlen > erofs_calc_inline_data_size(inode)) {
+ erofs_deinit_compress_context(&ctx);
+ return 0;
+ }
+
+#else
+ erofs_deinit_compress_context(&ctx);
+ return 0;
+#endif
+ } else {
+ ratio = ctx.cc_dstlen * 100 / ctx.cc_srclen;
+
+ if (ratio > erofs_cfg.c_compr_ratio_limit ||
+ ctx.cc_srclen - ctx.cc_dstlen < EROFS_BLKSIZE) {
+ erofs_deinit_compress_context(&ctx);
+ return 0;
+ }
+ }
+
+ /*
+ * Check the file compress ratio by trying to compress the 1st segment,
+ * If the ratio is greater than the limit or we can not save a block,
+ * don't compress.
+ */
+ inode->i_compressor.ci_alg = cinfo.ci_alg;
+ inode->i_compressor.ci_lvl = cinfo.ci_lvl;
+ memcpy(&inode->i_compr_ctx, &ctx, sizeof(struct erofs_compr_ctx));
+ return 1;
+}
+
diff --git a/mkfs_inode.c b/mkfs_inode.c
index e190853..4089e18 100644
--- a/mkfs_inode.c
+++ b/mkfs_inode.c
@@ -61,6 +61,23 @@ static inline u64 erofs_calc_compr_index_count(struct erofs_node_info *inode)
return round_up(inode->i_size, EROFS_BLKSIZE) / EROFS_BLKSIZE;
}
+static int erofs_calc_inline_compr_index_count(struct erofs_node_info *inode)
+{
+ int size;
+
+ size = erofs_calc_inode_base_size(inode);
+ size = round_up(size, EROFS_INLINE_INDEX_ALIGN_SIZE);
+ size += sizeof(struct erofs_extent_header);
+
+ assert(size < EROFS_BLKSIZE);
+
+ size = EROFS_BLKSIZE - size;
+
+ assert(size % EROFS_DECOMPR_IDX_SZ == 0);
+
+ return size / EROFS_DECOMPR_IDX_SZ;
+}
+
u8 erofs_check_disk_inode_version(struct erofs_node_info *inode)
{
#if 1
@@ -79,6 +96,33 @@ u8 erofs_check_disk_inode_version(struct erofs_node_info *inode)
#endif
}
+static void erofs_init_compress_inode(struct erofs_node_info *inode)
+{
+ int inlined_nidxs;
+
+ inode->i_dmode = EROFS_INODE_LAYOUT_COMPRESSION;
+
+ if (inode->i_compr_ctx.cc_nidxs == EROFS_COMPR_CTX_INLINED_DATA) {
+ inode->i_inline_datalen = inode->i_compr_ctx.cc_dstlen;
+ return;
+ }
+
+ inode->i_compr_nidxs = erofs_calc_compr_index_count(inode);
+
+ inlined_nidxs = erofs_calc_inline_compr_index_count(inode);
+
+ if (inode->i_compr_nidxs > (u64)inlined_nidxs)
+ inode->i_compr_inlined_nidxs = inlined_nidxs;
+ else
+ inode->i_compr_inlined_nidxs = inode->i_compr_nidxs;
+
+ inlined_nidxs = inode->i_compr_inlined_nidxs * EROFS_DECOMPR_IDX_SZ;
+ inode->i_inline_datalen = sizeof(struct erofs_extent_header);
+ inode->i_inline_datalen += inlined_nidxs;
+
+ inode->i_inline_align_size = EROFS_INLINE_INDEX_ALIGN_SIZE;
+}
+
void mkfs_rank_inode(struct erofs_node_info *inode)
{
block_buffer_t *blk;
@@ -201,6 +245,7 @@ struct erofs_node_info *mkfs_prepare_root_inode(char *root)
void mkfs_relocate_sub_inodes(struct erofs_node_info *inode)
{
+ int compressible;
u32 blkaddr;
u32 nblocks;
u32 unaligned;
@@ -208,6 +253,14 @@ void mkfs_relocate_sub_inodes(struct erofs_node_info *inode)
switch (d->i_type) {
case EROFS_FT_REG_FILE:
+ compressible = erofs_check_compressible(d);
+ if (compressible < 0) {
+ assert(0);
+ } else if (compressible > 0) {
+ erofs_init_compress_inode(d);
+ mkfs_rank_inode(d);
+ break;
+ }
case EROFS_FT_DIR:
case EROFS_FT_SYMLINK:
unaligned = d->i_size % EROFS_BLKSIZE;
@@ -488,8 +541,12 @@ static void mkfs_write_inode_regfile(struct erofs_node_info *inode)
break;
case EROFS_INODE_LAYOUT_COMPRESSION:
- /* it isn't supported right now */
- assert(0);
+ ret = erofs_compress_file(inode);
+ if (ret) {
+ erofs_err("Compress file failed");
+ exit(EXIT_FAILURE);
+ }
+ break;
case EROFS_INODE_LAYOUT_INLINE:
if (inode->i_size == 0)
diff --git a/mkfs_main.c b/mkfs_main.c
index 6e1423f..a334bb9 100644
--- a/mkfs_main.c
+++ b/mkfs_main.c
@@ -18,6 +18,7 @@
#include "mkfs_erofs.h"
#include "erofs_io.h"
#include "mkfs_inode.h"
+#include "erofs_compressor.h"
#define pr_fmt(fmt) "MKFS: "FUNC_LINE_FMT fmt"\n"
#include "erofs_debug.h"
@@ -107,6 +108,9 @@ static void mkfs_parse_options_cfg(int argc, char *argv[])
usage(argv[0]);
}
+ assert(erofs_cfg.c_alg_name);
+ erofs_compress_alg_init(erofs_cfg.c_alg_name);
+
mkfs_dump_config();
if (dev_open(erofs_cfg.c_img_path) < 0) {
--
2.17.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [PREVIEW] [RFC PATCH v2 5/5] erofs-mkfs: add miscellaneous files
2018-11-20 14:32 [PREVIEW] [RFC PATCH v2 0/5] erofs-mkfs: mkfs.erofs source code first preview Li Guifu
` (3 preceding siblings ...)
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 4/5] erofs-mkfs: add compression support Li Guifu
@ 2018-11-20 14:32 ` Li Guifu
4 siblings, 0 replies; 6+ messages in thread
From: Li Guifu @ 2018-11-20 14:32 UTC (permalink / raw)
Add build scripts, open source license and maintainers etc.
Signed-off-by: Li Guifu <bluce.liguifu at huawei.com>
Signed-off-by: Miao Xie <miaoxie at huawei.com>
Signed-off-by: Fang Wei <fangwei1 at huawei.com>
---
.gitignore | 44 +++++++
AUTHORS | 6 +
COPYING | 359 +++++++++++++++++++++++++++++++++++++++++++++++++++
ChangeLog | 0
Makefile.am | 35 +++++
NEWS | 0
README | 55 ++++++++
configure.ac | 49 +++++++
8 files changed, 548 insertions(+)
create mode 100644 .gitignore
create mode 100644 AUTHORS
create mode 100644 COPYING
create mode 100644 ChangeLog
create mode 100644 Makefile.am
create mode 100644 NEWS
create mode 100644 README
create mode 100644 configure.ac
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..7e0a1cf
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,44 @@
+autom4te.cache
+build
+build.profiled
+build.static
+FILES
+^core
+*~
+patches
+Makefile
+*.bak
+*.diff
+*.dSYM
+*.o
+*.a
+*.orig
+*.patch
+*.pc
+*.rej
+*.swp
+00[0-9][1-9]*.patch
+MCONFIG
+asm_types.h
+config.log
+config.status
+cscope.*
+tags
+TAGS
+.deps/
+Makefile.in
+aclocal.m4
+compile
+config.guess
+config.h
+config.h.in
+config.sub
+configure
+depcomp
+install-sh
+libtool
+ltmain.sh
+m4/
+missing
+stamp-h1
+
diff --git a/AUTHORS b/AUTHORS
new file mode 100644
index 0000000..d93f50d
--- /dev/null
+++ b/AUTHORS
@@ -0,0 +1,6 @@
+EROFS MKFS TOOL
+M: Li Guifu <bluce.liguifu at huawei.com>
+M: Fang Wei <fangwei1 at huawei.com>
+S: Maintained
+L: linux-erofs at lists.ozlabs.org
+
diff --git a/COPYING b/COPYING
new file mode 100644
index 0000000..ff0812f
--- /dev/null
+++ b/COPYING
@@ -0,0 +1,359 @@
+Valid-License-Identifier: GPL-2.0
+Valid-License-Identifier: GPL-2.0-only
+Valid-License-Identifier: GPL-2.0+
+Valid-License-Identifier: GPL-2.0-or-later
+SPDX-URL: https://spdx.org/licenses/GPL-2.0.html
+Usage-Guide:
+ To use this license in source code, put one of the following SPDX
+ tag/value pairs into a comment according to the placement
+ guidelines in the licensing rules documentation.
+ For 'GNU General Public License (GPL) version 2 only' use:
+ SPDX-License-Identifier: GPL-2.0
+ or
+ SPDX-License-Identifier: GPL-2.0-only
+ For 'GNU General Public License (GPL) version 2 or any later version' use:
+ SPDX-License-Identifier: GPL-2.0+
+ or
+ SPDX-License-Identifier: GPL-2.0-or-later
+License-Text:
+
+ GNU GENERAL PUBLIC LICENSE
+ Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.
+ 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The licenses for most software are designed to take away your
+freedom to share and change it. By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users. This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it. (Some other Free Software Foundation software is covered by
+the GNU Library General Public License instead.) You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+ To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have. You must make sure that they, too, receive or can get the
+source code. And you must show them these terms so they know their
+rights.
+
+ We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+ Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software. If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+ Finally, any free program is threatened constantly by software
+patents. We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary. To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+\f
+ GNU GENERAL PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License. The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language. (Hereinafter, translation is included without limitation in
+the term "modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope. The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+ 1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+ 2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+ a) You must cause the modified files to carry prominent notices
+ stating that you changed the files and the date of any change.
+
+ b) You must cause any work that you distribute or publish, that in
+ whole or in part contains or is derived from the Program or any
+ part thereof, to be licensed as a whole at no charge to all third
+ parties under the terms of this License.
+
+ c) If the modified program normally reads commands interactively
+ when run, you must cause it, when started running for such
+ interactive use in the most ordinary way, to print or display an
+ announcement including an appropriate copyright notice and a
+ notice that there is no warranty (or else, saying that you provide
+ a warranty) and that users may redistribute the program under
+ these conditions, and telling the user how to view a copy of this
+ License. (Exception: if the Program itself is interactive but
+ does not normally print such an announcement, your work based on
+ the Program is not required to print an announcement.)
+\f
+These requirements apply to the modified work as a whole. If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works. But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+ 3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+ a) Accompany it with the complete corresponding machine-readable
+ source code, which must be distributed under the terms of Sections
+ 1 and 2 above on a medium customarily used for software interchange; or,
+
+ b) Accompany it with a written offer, valid for at least three
+ years, to give any third party, for a charge no more than your
+ cost of physically performing source distribution, a complete
+ machine-readable copy of the corresponding source code, to be
+ distributed under the terms of Sections 1 and 2 above on a medium
+ customarily used for software interchange; or,
+
+ c) Accompany it with the information you received as to the offer
+ to distribute corresponding source code. (This alternative is
+ allowed only for noncommercial distribution and only if you
+ received the program in object code or executable form with such
+ an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it. For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable. However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+\f
+ 4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License. Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+ 5. You are not required to accept this License, since you have not
+signed it. However, nothing else grants you permission to modify or
+distribute the Program or its derivative works. These actions are
+prohibited by law if you do not accept this License. Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+ 6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions. You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+ 7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all. For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices. Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+\f
+ 8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded. In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+ 9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation. If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+ 10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission. For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this. Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+ NO WARRANTY
+
+ 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+ 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+ END OF TERMS AND CONDITIONS
+\f
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+ Gnomovision version 69, Copyright (C) year name of author
+ Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. Here is a sample; alter the names:
+
+ Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+ `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+ <signature of Ty Coon>, 1 April 1989
+ Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs. If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library. If this is what you want to do, use the GNU Library General
+Public License instead of this License.
diff --git a/ChangeLog b/ChangeLog
new file mode 100644
index 0000000..e69de29
diff --git a/Makefile.am b/Makefile.am
new file mode 100644
index 0000000..b3dc472
--- /dev/null
+++ b/Makefile.am
@@ -0,0 +1,35 @@
+# Makefile.am
+AUTOMAKE_OPTIONS = foreign
+bin_PROGRAMS = mkfs.erofs
+mkfs_erofs_SOURCES = erofs_compressor.c \
+ mkfs_file.c \
+ erofs_lz4hc.c \
+ mkfs_main.c \
+ erofs_cache.c \
+ erofs_io.c \
+ mkfs_inode.c \
+ erofs_config.c
+
+noinst_HEADERS = erofs_config.h \
+ mkfs_inode.h \
+ erofs_error.h \
+ erofs_lz4hc.h \
+ list_head.h \
+ erofs_cache.h \
+ erofs_fs.h \
+ erofs_debug.h \
+ mkfs_erofs.h \
+ erofs_compressor.h \
+ erofs_io.h \
+ erofs_types.h \
+ mkfs_file.h
+
+mkfs_erofs_CFLAGS = -Wall -Werror -DEROFS_MKFS_VERSION=\"v1.0\"
+mkfs_erofs_LDFLAGS = --static
+mkfs_erofs_LDADD = $(LIBLZ4_STATIC)
+ACLOCAL_AMFLAGS = -I m4
+
+if SUPPORT_LARG_FILE_AT_BIT32
+ mkfs_erofs_CFLAGS += -D_FILE_OFFSET_BITS=64
+endif
+
diff --git a/NEWS b/NEWS
new file mode 100644
index 0000000..e69de29
diff --git a/README b/README
new file mode 100644
index 0000000..79a1126
--- /dev/null
+++ b/README
@@ -0,0 +1,55 @@
+mkfs.erofs
+
+mkfs.erofs is a user-space tool to create erofs filesystem images.
+
+It can create 2 main types of erofs images, compressed and uncompressed:
+ - For compressed images, it is able to integrate several compression
+ algorithms, lz4 is supported according to the current erofs kernel
+ implementation.
+ - For uncompressed images, it can decide whether the last page of
+ a file should be inlined or not properly [1].
+
+Note that mkfs.erofs can only be linked statically with lz4 library
+due to the dependency of experimental lz4hc apis which were added in
+lz4 1.8.0. Anyway, it's a good start to begin with building the latest
+lz4 library statically first. :)
+
+Dependencies
+ lz4-1.8.0 or above
+
+How to build with lz4 static library
+ ./configure --with-lz4=<lz4 install path>
+eg. if lz4 lib has been installed into fold of /usr/local/lib
+ ./configure --with-lz4=/usr/local/lib && make
+Maybe you should run this first:
+ libtoolize && aclocal && autoconf && autoheader && automake --add-missing
+
+Usage:
+$ ./mkfs.erofs
+mkfs.erofs v1.0 Nov 17 2018 19:47:21
+
+Usage:
+ [-z <compr_algri>] [-d <dbglvl>]
+ [target path] [source directory]
+
+ -d -- set debugging level <dbglvl>
+ -z -- enable <compr_algri> compression (only lz4hc is supported)
+
+ target path, source directory are both needed.
+
+To:
+ linux-erofs mailing list <linux-erofs at lists.ozlabs.org>
+ Li Guifu <bluce.liguifu at huawei.com>
+ Miao Xie <miaoxie at huawei.com>
+ Fang Wei <fangwei1 at huawei.com>
+
+Thanks-to:
+ Sun Qiuyang <sunqiuyang at huawei.com>
+ Guo Xuenan <guoxuenan at huawei.com>
+ Du Wei <weidu.du at huawei.com>
+ Wang Zhigang <brooke.wangzhigang at hisilicon.com>
+
+[1] According to the erofs on-disk format, the last page of files could
+ be inlined aggressively with its metadata in order to reduce the I/O
+ overhead and save the storage space.
+
diff --git a/configure.ac b/configure.ac
new file mode 100644
index 0000000..15974f8
--- /dev/null
+++ b/configure.ac
@@ -0,0 +1,49 @@
+# -*- Autoconf -*-
+# Process this file with autoconf to produce a configure script.
+
+AC_PREREQ([2.69])
+AC_INIT([mkfs.erofs], [0.0.1], [bluce.liguifu at huawei.com])
+AM_INIT_AUTOMAKE([foreign -Wall -Werror])
+AC_CONFIG_SRCDIR([mkfs_main.c])
+AC_CONFIG_HEADERS([config.h])
+
+# Checks for programs.
+AC_PROG_CC
+AC_PROG_INSTALL
+AC_PROG_LIBTOOL
+
+
+# checks system architecture firtly for enable support larg file
+ARCH_BIT=`getconf LONG_BIT`
+AM_CONDITIONAL([SUPPORT_LARG_FILE_AT_BIT32],[test x$ARCH_BIT = x32])
+
+# Checks for libraries.
+# Ask user for path to liblz4.a stuff:.
+AC_ARG_WITH(lz4,
+ [ --with-lz4=<path> prefix of liblz4.a installation. e.g. /usr/local or /usr],
+ [LIBLZ4_STATIC=$with_lz4/liblz4.a],
+ AC_MSG_ERROR([You must call configure with the --with-lz4 option.
+ This tells configure where to find the MySql C library and headers.
+ e.g. --with-lz4=/usr/local or --with-lz4=/usr])
+)
+
+AC_SUBST(LIBLZ4_STATIC)
+
+# Checks for header files.
+AC_CHECK_HEADERS([fcntl.h inttypes.h limits.h stddef.h stdint.h stdlib.h string.h sys/time.h unistd.h])
+
+
+# Checks for typedefs, structures, and compiler characteristics.
+AC_C_INLINE
+AC_TYPE_INT64_T
+AC_TYPE_SIZE_T
+AC_TYPE_SSIZE_T
+AC_CHECK_MEMBERS([struct stat.st_rdev])
+AC_TYPE_UINT64_T
+
+# Checks for library functions.
+AC_FUNC_MALLOC
+AC_CHECK_FUNCS([ftruncate getcwd gettimeofday memset realpath strdup strerror strrchr strtoull])
+
+AC_CONFIG_FILES([Makefile])
+AC_OUTPUT
--
2.17.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
end of thread, other threads:[~2018-11-20 14:32 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2018-11-20 14:32 [PREVIEW] [RFC PATCH v2 0/5] erofs-mkfs: mkfs.erofs source code first preview Li Guifu
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 1/5] erofs-mkfs: add erofs on-disk layout Li Guifu
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 2/5] erofs-mkfs: introduce mkfs basic framework Li Guifu
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 3/5] erofs-mkfs: support to build a image from the specific root directory Li Guifu
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 4/5] erofs-mkfs: add compression support Li Guifu
2018-11-20 14:32 ` [PREVIEW] [RFC PATCH v2 5/5] erofs-mkfs: add miscellaneous files Li Guifu
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.