* [PATCH v4 1/6] erofs-utils: add extra device I/O interface
2021-11-16 9:49 [PATCH v4 0/6] erofs-utils: add multiple device support Gao Xiang
@ 2021-11-16 9:49 ` Gao Xiang
2021-11-16 9:49 ` [PATCH v4 2/6] erofs-utils: fuse: add multiple device support Gao Xiang
` (4 subsequent siblings)
5 siblings, 0 replies; 9+ messages in thread
From: Gao Xiang @ 2021-11-16 9:49 UTC (permalink / raw)
To: linux-erofs
Cc: Yan Song, Peng Tao, Joseph Qi, Liu Bo, Changwei Ge, Gao Xiang, Liu Jiang
In order for erofsfuse to support multiple devices.
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
fsck/main.c | 8 ++++----
include/erofs/io.h | 10 ++++++----
lib/data.c | 6 +++---
lib/io.c | 48 +++++++++++++++++++++++++++++++++++++---------
lib/namei.c | 4 ++--
lib/super.c | 2 +-
lib/zmap.c | 4 ++--
mkfs/main.c | 2 +-
8 files changed, 58 insertions(+), 26 deletions(-)
diff --git a/fsck/main.c b/fsck/main.c
index d81d60024c8a..b742e3579c59 100644
--- a/fsck/main.c
+++ b/fsck/main.c
@@ -97,7 +97,7 @@ static int erofs_check_sb_chksum(void)
u32 crc;
struct erofs_super_block *sb;
- ret = blk_read(buf, 0, 1);
+ ret = blk_read(0, buf, 0, 1);
if (ret) {
erofs_err("failed to read superblock to check checksum: %d",
ret);
@@ -317,7 +317,7 @@ static int verify_compressed_inode(struct erofs_inode *inode)
BUG_ON(!buffer);
}
- ret = dev_read(raw, map.m_pa, map.m_plen);
+ ret = dev_read(0, raw, map.m_pa, map.m_plen);
if (ret < 0) {
erofs_err("failed to read compressed data of m_pa %" PRIu64 ", m_plen %" PRIu64 " @ nid %llu: %d",
map.m_pa, map.m_plen, inode->nid | 0ULL, ret);
@@ -381,7 +381,7 @@ static int erofs_verify_xattr(struct erofs_inode *inode)
}
addr = iloc(inode->nid) + inode->inode_isize;
- ret = dev_read(buf, addr, xattr_hdr_size);
+ ret = dev_read(0, buf, addr, xattr_hdr_size);
if (ret < 0) {
erofs_err("failed to read xattr header @ nid %llu: %d",
inode->nid | 0ULL, ret);
@@ -411,7 +411,7 @@ static int erofs_verify_xattr(struct erofs_inode *inode)
while (remaining > 0) {
unsigned int entry_sz;
- ret = dev_read(buf, addr, xattr_entry_size);
+ ret = dev_read(0, buf, addr, xattr_entry_size);
if (ret) {
erofs_err("failed to read xattr entry @ nid %llu: %d",
inode->nid | 0ULL, ret);
diff --git a/include/erofs/io.h b/include/erofs/io.h
index 9d73adc5f5f9..10a3681882e1 100644
--- a/include/erofs/io.h
+++ b/include/erofs/io.h
@@ -15,11 +15,13 @@
#define O_BINARY 0
#endif
+void blob_closeall(void);
+int blob_open_ro(const char *dev);
int dev_open(const char *devname);
int dev_open_ro(const char *dev);
void dev_close(void);
int dev_write(const void *buf, u64 offset, size_t len);
-int dev_read(void *buf, u64 offset, size_t len);
+int dev_read(int device_id, void *buf, u64 offset, size_t len);
int dev_fillzero(u64 offset, size_t len, bool padding);
int dev_fsync(void);
int dev_resize(erofs_blk_t nblocks);
@@ -38,10 +40,10 @@ static inline int blk_write(const void *buf, erofs_blk_t blkaddr,
blknr_to_addr(nblocks));
}
-static inline int blk_read(void *buf, erofs_blk_t start,
- u32 nblocks)
+static inline int blk_read(int device_id, void *buf,
+ erofs_blk_t start, u32 nblocks)
{
- return dev_read(buf, blknr_to_addr(start),
+ return dev_read(device_id, buf, blknr_to_addr(start),
blknr_to_addr(nblocks));
}
diff --git a/lib/data.c b/lib/data.c
index b5f0196c97dd..b83cbff3e731 100644
--- a/lib/data.c
+++ b/lib/data.c
@@ -91,7 +91,7 @@ int erofs_map_blocks(struct erofs_inode *inode,
pos = roundup(iloc(vi->nid) + vi->inode_isize +
vi->xattr_isize, unit) + unit * chunknr;
- err = blk_read(buf, erofs_blknr(pos), 1);
+ err = blk_read(0, buf, erofs_blknr(pos), 1);
if (err < 0)
return -EIO;
@@ -176,7 +176,7 @@ static int erofs_read_raw_data(struct erofs_inode *inode, char *buffer,
map.m_la = ptr;
}
- ret = dev_read(estart, map.m_pa, eend - map.m_la);
+ ret = dev_read(0, estart, map.m_pa, eend - map.m_la);
if (ret < 0)
return -EIO;
ptr = eend;
@@ -240,7 +240,7 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,
break;
}
}
- ret = dev_read(raw, map.m_pa, map.m_plen);
+ ret = dev_read(0, raw, map.m_pa, map.m_plen);
if (ret < 0)
break;
diff --git a/lib/io.c b/lib/io.c
index 279c7dd4b877..a0d366a4c3f1 100644
--- a/lib/io.c
+++ b/lib/io.c
@@ -26,6 +26,7 @@
static const char *erofs_devname;
int erofs_devfd = -1;
static u64 erofs_devsz;
+static unsigned int erofs_nblobs, erofs_blobfd[256];
int dev_get_blkdev_size(int fd, u64 *bytes)
{
@@ -106,6 +107,30 @@ int dev_open(const char *dev)
return 0;
}
+void blob_closeall(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < erofs_nblobs; ++i)
+ close(erofs_blobfd[i]);
+ erofs_nblobs = 0;
+}
+
+int blob_open_ro(const char *dev)
+{
+ int fd = open(dev, O_RDONLY | O_BINARY);
+
+ if (fd < 0) {
+ erofs_err("failed to open(%s).", dev);
+ return -errno;
+ }
+
+ erofs_blobfd[erofs_nblobs] = fd;
+ erofs_info("successfully to open blob%u %s", erofs_nblobs, dev);
+ ++erofs_nblobs;
+ return 0;
+}
+
/* XXX: temporary soluation. Disk I/O implementation needs to be refactored. */
int dev_open_ro(const char *dev)
{
@@ -229,9 +254,9 @@ int dev_resize(unsigned int blocks)
return dev_fillzero(st.st_size, length, true);
}
-int dev_read(void *buf, u64 offset, size_t len)
+int dev_read(int device_id, void *buf, u64 offset, size_t len)
{
- int ret;
+ int ret, fd;
if (cfg.c_dry_run)
return 0;
@@ -240,16 +265,21 @@ int dev_read(void *buf, u64 offset, size_t len)
erofs_err("buf is NULL");
return -EINVAL;
}
- if (offset >= erofs_devsz || len > erofs_devsz ||
- offset > erofs_devsz - len) {
- erofs_err("read posion[%" PRIu64 ", %zd] is too large beyond the end of device(%" PRIu64 ").",
- offset, len, erofs_devsz);
- return -EINVAL;
+
+ if (!device_id) {
+ fd = erofs_devfd;
+ } else {
+ if (device_id > erofs_nblobs) {
+ erofs_err("invalid device id %d", device_id);
+ return -ENODEV;
+ }
+ fd = erofs_blobfd[device_id - 1];
}
+
#ifdef HAVE_PREAD64
- ret = pread64(erofs_devfd, buf, len, (off64_t)offset);
+ ret = pread64(fd, buf, len, (off64_t)offset);
#else
- ret = pread(erofs_devfd, buf, len, (off_t)offset);
+ ret = pread(fd, buf, len, (off_t)offset);
#endif
if (ret != (int)len) {
erofs_err("Failed to read data from device - %s:[%" PRIu64 ", %zd].",
diff --git a/lib/namei.c b/lib/namei.c
index 56f199aba382..7377e74fad9b 100644
--- a/lib/namei.c
+++ b/lib/namei.c
@@ -30,7 +30,7 @@ int erofs_read_inode_from_disk(struct erofs_inode *vi)
struct erofs_inode_extended *die;
const erofs_off_t inode_loc = iloc(vi->nid);
- ret = dev_read(buf, inode_loc, sizeof(*dic));
+ ret = dev_read(0, buf, inode_loc, sizeof(*dic));
if (ret < 0)
return -EIO;
@@ -47,7 +47,7 @@ int erofs_read_inode_from_disk(struct erofs_inode *vi)
case EROFS_INODE_LAYOUT_EXTENDED:
vi->inode_isize = sizeof(struct erofs_inode_extended);
- ret = dev_read(buf + sizeof(*dic), inode_loc + sizeof(*dic),
+ ret = dev_read(0, buf + sizeof(*dic), inode_loc + sizeof(*dic),
sizeof(*die) - sizeof(*dic));
if (ret < 0)
return -EIO;
diff --git a/lib/super.c b/lib/super.c
index 0c3040394272..bf4d4318a321 100644
--- a/lib/super.c
+++ b/lib/super.c
@@ -30,7 +30,7 @@ int erofs_read_superblock(void)
unsigned int blkszbits;
int ret;
- ret = blk_read(data, 0, 1);
+ ret = blk_read(0, data, 0, 1);
if (ret < 0) {
erofs_err("cannot read erofs superblock: %d", ret);
return -EIO;
diff --git a/lib/zmap.c b/lib/zmap.c
index 7dbda87c3b13..9dd0c7633a45 100644
--- a/lib/zmap.c
+++ b/lib/zmap.c
@@ -38,7 +38,7 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode *vi)
vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY);
pos = round_up(iloc(vi->nid) + vi->inode_isize + vi->xattr_isize, 8);
- ret = dev_read(buf, pos, sizeof(buf));
+ ret = dev_read(0, buf, pos, sizeof(buf));
if (ret < 0)
return -EIO;
@@ -88,7 +88,7 @@ static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m,
if (map->index == eblk)
return 0;
- ret = blk_read(mpage, eblk, 1);
+ ret = blk_read(0, mpage, eblk, 1);
if (ret < 0)
return -EIO;
diff --git a/mkfs/main.c b/mkfs/main.c
index 4ea5467679b7..2604bf2abd6b 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -436,7 +436,7 @@ static int erofs_mkfs_superblock_csum_set(void)
u32 crc;
struct erofs_super_block *sb;
- ret = blk_read(buf, 0, 1);
+ ret = blk_read(0, buf, 0, 1);
if (ret) {
erofs_err("failed to read superblock to set checksum: %s",
erofs_strerror(ret));
--
2.24.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v4 2/6] erofs-utils: fuse: add multiple device support
2021-11-16 9:49 [PATCH v4 0/6] erofs-utils: add multiple device support Gao Xiang
2021-11-16 9:49 ` [PATCH v4 1/6] erofs-utils: add extra device I/O interface Gao Xiang
@ 2021-11-16 9:49 ` Gao Xiang
2021-11-16 9:49 ` [PATCH v4 3/6] erofs-utils: mkfs: add extra blob " Gao Xiang
` (3 subsequent siblings)
5 siblings, 0 replies; 9+ messages in thread
From: Gao Xiang @ 2021-11-16 9:49 UTC (permalink / raw)
To: linux-erofs
Cc: Yan Song, Peng Tao, Joseph Qi, Liu Bo, Changwei Ge, Gao Xiang, Liu Jiang
Keep in sync with the latest kernel
commit dfeab2e95a75 ("erofs: add multiple device support")
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
dump/main.c | 2 +-
fuse/main.c | 11 +++++++
include/erofs/defs.h | 32 +++++++++++++++++++
include/erofs/internal.h | 19 +++++++++++-
include/erofs_fs.h | 22 ++++++++++++--
lib/data.c | 66 +++++++++++++++++++++++++++++++++-------
lib/super.c | 43 ++++++++++++++++++++++++--
man/erofsfuse.1 | 4 +++
8 files changed, 181 insertions(+), 18 deletions(-)
diff --git a/dump/main.c b/dump/main.c
index a7199937b8e0..d0efe9505317 100644
--- a/dump/main.c
+++ b/dump/main.c
@@ -611,7 +611,7 @@ static void erofsdump_show_superblock(void)
fprintf(stdout, "Filesystem magic number: 0x%04X\n",
EROFS_SUPER_MAGIC_V1);
fprintf(stdout, "Filesystem blocks: %llu\n",
- sbi.blocks | 0ULL);
+ sbi.total_blocks | 0ULL);
fprintf(stdout, "Filesystem inode metadata start block: %u\n",
sbi.meta_blkaddr);
fprintf(stdout, "Filesystem shared xattr metadata start block: %u\n",
diff --git a/fuse/main.c b/fuse/main.c
index 8137421b1dea..a92f06882b75 100644
--- a/fuse/main.c
+++ b/fuse/main.c
@@ -113,6 +113,7 @@ static struct options {
static const struct fuse_opt option_spec[] = {
OPTION("--dbglevel=%u", debug_lvl),
OPTION("--help", show_help),
+ FUSE_OPT_KEY("--device=", 1),
FUSE_OPT_END
};
@@ -123,6 +124,7 @@ static void usage(void)
fputs("usage: [options] IMAGE MOUNTPOINT\n\n"
"Options:\n"
" --dbglevel=# set output message level to # (maximum 9)\n"
+ " --device=# specify an extra device to be used together\n"
#if FUSE_MAJOR_VERSION < 3
" --help display this help and exit\n"
#endif
@@ -148,7 +150,15 @@ static void erofsfuse_dumpcfg(void)
static int optional_opt_func(void *data, const char *arg, int key,
struct fuse_args *outargs)
{
+ int ret;
+
switch (key) {
+ case 1:
+ ret = blob_open_ro(arg);
+ if (ret)
+ return -1;
+ ++sbi.extra_devices;
+ return 0;
case FUSE_OPT_KEY_NONOPT:
if (fusecfg.mountpoint)
return -1; /* Too many args */
@@ -237,6 +247,7 @@ int main(int argc, char *argv[])
ret = fuse_main(args.argc, args.argv, &erofs_ops, NULL);
err_dev_close:
+ blob_closeall();
dev_close();
err_fuse_free_args:
fuse_opt_free_args(&args);
diff --git a/include/erofs/defs.h b/include/erofs/defs.h
index 96bbb6574ff3..6398cbb2aa4d 100644
--- a/include/erofs/defs.h
+++ b/include/erofs/defs.h
@@ -252,6 +252,38 @@ static inline u32 get_unaligned_le32(const u8 *p)
(n) & (1ULL << 1) ? 1 : 0 \
)
+static inline unsigned int fls_long(unsigned long x)
+{
+ return x ? sizeof(x) * 8 - __builtin_clz(x) : 0;
+}
+
+/**
+ * __roundup_pow_of_two() - round up to nearest power of two
+ * @n: value to round up
+ */
+static inline __attribute__((const))
+unsigned long __roundup_pow_of_two(unsigned long n)
+{
+ return 1UL << fls_long(n - 1);
+}
+
+/**
+ * roundup_pow_of_two - round the given value up to nearest power of two
+ * @n: parameter
+ *
+ * round the given value up to the nearest power of two
+ * - the result is undefined when n == 0
+ * - this can be used to initialise global variables from constant data
+ */
+#define roundup_pow_of_two(n) \
+( \
+ __builtin_constant_p(n) ? ( \
+ ((n) == 1) ? 1 : \
+ (1UL << (ilog2((n) - 1) + 1)) \
+ ) : \
+ __roundup_pow_of_two(n) \
+)
+
#ifndef __always_inline
#define __always_inline inline
#endif
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index f84e6b4f125d..974c069baa4f 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -60,8 +60,16 @@ typedef u32 erofs_blk_t;
struct erofs_buffer_head;
+struct erofs_device_info {
+ u32 blocks;
+ u32 mapped_blkaddr;
+};
+
struct erofs_sb_info {
- u64 blocks;
+ struct erofs_device_info *devs;
+
+ u64 total_blocks;
+ u64 primarydevice_blocks;
erofs_blk_t meta_blkaddr;
erofs_blk_t xattr_blkaddr;
@@ -84,6 +92,8 @@ struct erofs_sb_info {
u16 lz4_max_distance;
u32 checksum;
+ u16 extra_devices;
+ u16 device_id_mask;
};
/* global sbi */
@@ -112,6 +122,7 @@ EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)
EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)
EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)
+EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE)
EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
#define EROFS_I_EA_INITED (1 << 0)
@@ -257,6 +268,7 @@ struct erofs_map_blocks {
erofs_off_t m_pa, m_la;
u64 m_plen, m_llen;
+ unsigned short m_deviceid;
unsigned int m_flags;
erofs_blk_t index;
};
@@ -267,6 +279,11 @@ struct erofs_map_blocks {
*/
#define EROFS_GET_BLOCKS_FIEMAP 0x0002
+struct erofs_map_dev {
+ erofs_off_t m_pa;
+ unsigned int m_deviceid;
+};
+
/* super.c */
int erofs_read_superblock(void);
diff --git a/include/erofs_fs.h b/include/erofs_fs.h
index 4291970753a8..9a918775750c 100644
--- a/include/erofs_fs.h
+++ b/include/erofs_fs.h
@@ -22,14 +22,27 @@
#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002
#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002
#define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004
+#define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 0x00000008
#define EROFS_ALL_FEATURE_INCOMPAT \
(EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \
EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \
EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \
- EROFS_FEATURE_INCOMPAT_CHUNKED_FILE)
+ EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \
+ EROFS_FEATURE_INCOMPAT_DEVICE_TABLE)
#define EROFS_SB_EXTSLOT_SIZE 16
+struct erofs_deviceslot {
+ union {
+ u8 uuid[16]; /* used for device manager later */
+ u8 userdata[64]; /* digest(sha256), etc. */
+ } u;
+ __le32 blocks; /* total fs blocks of this device */
+ __le32 mapped_blkaddr; /* map starting at mapped_blkaddr */
+ u8 reserved[56];
+};
+#define EROFS_DEVT_SLOT_SIZE sizeof(struct erofs_deviceslot)
+
/* erofs on-disk super block (currently 128 bytes) */
struct erofs_super_block {
__le32 magic; /* file system magic number */
@@ -55,7 +68,9 @@ struct erofs_super_block {
/* customized sliding window size instead of 64k by default */
__le16 lz4_max_distance;
} __packed u1;
- __u8 reserved2[42];
+ __le16 extra_devices; /* # of devices besides the primary device */
+ __le16 devt_slotoff; /* startoff = devt_slotoff * devt_slotsize */
+ __u8 reserved2[38];
};
/*
@@ -239,7 +254,7 @@ static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e)
/* 8-byte inode chunk indexes */
struct erofs_inode_chunk_index {
__le16 advise; /* always 0, don't care for now */
- __le16 device_id; /* back-end storage id, always 0 for now */
+ __le16 device_id; /* back-end storage id (with bits masked) */
__le32 blkaddr; /* start block address of this inode chunk */
};
@@ -404,6 +419,7 @@ static inline void erofs_check_ondisk_layout_definitions(void)
/* keep in sync between 2 index structures for better extendibility */
BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_index) !=
sizeof(struct z_erofs_vle_decompressed_index));
+ BUILD_BUG_ON(sizeof(struct erofs_deviceslot) != 128);
BUILD_BUG_ON(BIT(Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) <
Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1);
diff --git a/lib/data.c b/lib/data.c
index b83cbff3e731..136c0d97ab45 100644
--- a/lib/data.c
+++ b/lib/data.c
@@ -72,6 +72,7 @@ int erofs_map_blocks(struct erofs_inode *inode,
erofs_off_t pos;
int err = 0;
+ map->m_deviceid = 0;
if (map->m_la >= inode->i_size) {
/* leave out-of-bound access unmapped */
map->m_flags = 0;
@@ -118,14 +119,8 @@ int erofs_map_blocks(struct erofs_inode *inode,
map->m_flags = 0;
break;
default:
- /* only one device is supported for now */
- if (idx->device_id) {
- erofs_err("invalid device id %u @ %" PRIu64 " for nid %llu",
- le16_to_cpu(idx->device_id),
- chunknr, vi->nid | 0ULL);
- err = -EFSCORRUPTED;
- goto out;
- }
+ map->m_deviceid = le16_to_cpu(idx->device_id) &
+ sbi.device_id_mask;
map->m_pa = blknr_to_addr(le32_to_cpu(idx->blkaddr));
map->m_flags = EROFS_MAP_MAPPED;
break;
@@ -135,12 +130,41 @@ out:
return err;
}
+int erofs_map_dev(struct erofs_sb_info *sbi, struct erofs_map_dev *map)
+{
+ struct erofs_device_info *dif;
+ int id;
+
+ if (map->m_deviceid) {
+ if (sbi->extra_devices < map->m_deviceid)
+ return -ENODEV;
+ } else if (sbi->extra_devices) {
+ for (id = 0; id < sbi->extra_devices; ++id) {
+ erofs_off_t startoff, length;
+
+ dif = sbi->devs + id;
+ if (!dif->mapped_blkaddr)
+ continue;
+ startoff = blknr_to_addr(dif->mapped_blkaddr);
+ length = blknr_to_addr(dif->blocks);
+
+ if (map->m_pa >= startoff &&
+ map->m_pa < startoff + length) {
+ map->m_pa -= startoff;
+ break;
+ }
+ }
+ }
+ return 0;
+}
+
static int erofs_read_raw_data(struct erofs_inode *inode, char *buffer,
erofs_off_t size, erofs_off_t offset)
{
struct erofs_map_blocks map = {
.index = UINT_MAX,
};
+ struct erofs_map_dev mdev;
int ret;
erofs_off_t ptr = offset;
@@ -155,6 +179,14 @@ static int erofs_read_raw_data(struct erofs_inode *inode, char *buffer,
DBG_BUGON(map.m_plen != map.m_llen);
+ mdev = (struct erofs_map_dev) {
+ .m_deviceid = map.m_deviceid,
+ .m_pa = map.m_pa,
+ };
+ ret = erofs_map_dev(&sbi, &mdev);
+ if (ret)
+ return ret;
+
/* trim extent */
eend = min(offset + size, map.m_la + map.m_llen);
DBG_BUGON(ptr < map.m_la);
@@ -172,11 +204,12 @@ static int erofs_read_raw_data(struct erofs_inode *inode, char *buffer,
}
if (ptr > map.m_la) {
- map.m_pa += ptr - map.m_la;
+ mdev.m_pa += ptr - map.m_la;
map.m_la = ptr;
}
- ret = dev_read(0, estart, map.m_pa, eend - map.m_la);
+ ret = dev_read(mdev.m_deviceid, estart, mdev.m_pa,
+ eend - map.m_la);
if (ret < 0)
return -EIO;
ptr = eend;
@@ -191,6 +224,7 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,
struct erofs_map_blocks map = {
.index = UINT_MAX,
};
+ struct erofs_map_dev mdev;
bool partial;
unsigned int algorithmformat, bufsize;
char *raw = NULL;
@@ -205,6 +239,16 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,
if (ret)
break;
+ /* no device id here, thus it will always succeed */
+ mdev = (struct erofs_map_dev) {
+ .m_pa = map.m_pa,
+ };
+ ret = erofs_map_dev(&sbi, &mdev);
+ if (ret) {
+ DBG_BUGON(1);
+ break;
+ }
+
/*
* trim to the needed size if the returned extent is quite
* larger than requested, and set up partial flag as well.
@@ -240,7 +284,7 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,
break;
}
}
- ret = dev_read(0, raw, map.m_pa, map.m_plen);
+ ret = dev_read(mdev.m_deviceid, raw, mdev.m_pa, map.m_plen);
if (ret < 0)
break;
diff --git a/lib/super.c b/lib/super.c
index bf4d4318a321..3ccc551f18cf 100644
--- a/lib/super.c
+++ b/lib/super.c
@@ -23,6 +23,45 @@ static bool check_layout_compatibility(struct erofs_sb_info *sbi,
return true;
}
+static int erofs_init_devices(struct erofs_sb_info *sbi,
+ struct erofs_super_block *dsb)
+{
+ unsigned int ondisk_extradevs, i;
+ erofs_off_t pos;
+
+ sbi->total_blocks = sbi->primarydevice_blocks;
+
+ if (!erofs_sb_has_device_table())
+ ondisk_extradevs = 0;
+ else
+ ondisk_extradevs = le16_to_cpu(dsb->extra_devices);
+
+ if (ondisk_extradevs != sbi->extra_devices) {
+ erofs_err("extra devices don't match (ondisk %u, given %u)",
+ ondisk_extradevs, sbi->extra_devices);
+ return -EINVAL;
+ }
+ if (!ondisk_extradevs)
+ return 0;
+
+ sbi->device_id_mask = roundup_pow_of_two(ondisk_extradevs + 1) - 1;
+ sbi->devs = calloc(ondisk_extradevs, sizeof(*sbi->devs));
+ pos = le16_to_cpu(dsb->devt_slotoff) * EROFS_DEVT_SLOT_SIZE;
+ for (i = 0; i < ondisk_extradevs; ++i) {
+ struct erofs_deviceslot dis;
+ int ret;
+
+ ret = dev_read(0, &dis, pos, sizeof(dis));
+ if (ret < 0)
+ return ret;
+
+ sbi->devs[i].mapped_blkaddr = dis.mapped_blkaddr;
+ sbi->total_blocks += dis.blocks;
+ pos += EROFS_DEVT_SLOT_SIZE;
+ }
+ return 0;
+}
+
int erofs_read_superblock(void)
{
char data[EROFS_BLKSIZ];
@@ -56,7 +95,7 @@ int erofs_read_superblock(void)
if (!check_layout_compatibility(&sbi, dsb))
return ret;
- sbi.blocks = le32_to_cpu(dsb->blocks);
+ sbi.primarydevice_blocks = le32_to_cpu(dsb->blocks);
sbi.meta_blkaddr = le32_to_cpu(dsb->meta_blkaddr);
sbi.xattr_blkaddr = le32_to_cpu(dsb->xattr_blkaddr);
sbi.islotbits = EROFS_ISLOTBITS;
@@ -68,5 +107,5 @@ int erofs_read_superblock(void)
sbi.build_time_nsec = le32_to_cpu(dsb->build_time_nsec);
memcpy(&sbi.uuid, dsb->uuid, sizeof(dsb->uuid));
- return 0;
+ return erofs_init_devices(&sbi, dsb);
}
diff --git a/man/erofsfuse.1 b/man/erofsfuse.1
index 6bd48b0460bd..9db6827f4d0e 100644
--- a/man/erofsfuse.1
+++ b/man/erofsfuse.1
@@ -22,6 +22,10 @@ display help and exit
.BI "\-\-dbglevel=" #
Specify the level of debugging messages. The default is 2, which shows basic
warning messages.
+.TP
+.BI "\-\-device=" path
+Specify an extra device to be used together.
+You may give multiple `--device' options in the correct order.
.SS "FUSE options:"
.TP
\fB-d -o\fR debug
--
2.24.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v4 3/6] erofs-utils: mkfs: add extra blob device support
2021-11-16 9:49 [PATCH v4 0/6] erofs-utils: add multiple device support Gao Xiang
2021-11-16 9:49 ` [PATCH v4 1/6] erofs-utils: add extra device I/O interface Gao Xiang
2021-11-16 9:49 ` [PATCH v4 2/6] erofs-utils: fuse: add multiple device support Gao Xiang
@ 2021-11-16 9:49 ` Gao Xiang
2021-11-16 9:49 ` [PATCH v4 4/6] erofs-utils: dump: support multiple devices Gao Xiang
` (2 subsequent siblings)
5 siblings, 0 replies; 9+ messages in thread
From: Gao Xiang @ 2021-11-16 9:49 UTC (permalink / raw)
To: linux-erofs
Cc: Yan Song, Peng Tao, Joseph Qi, Liu Bo, Changwei Ge, Gao Xiang, Liu Jiang
In this patch, blob data from chunked-based files is redirected to
another blob file.
In order to achieve that, "--blobdev" should be used to specify
the output blob file/device for all chunk-based files, e.g.
mkfs.erofs --blobdev blob.erofs --chunksize 4096 foo.erofs foo
Note that the upcoming RAFS v6 (EROFS-compatible on-disk format) [1]
will make full use of EROFS multiple device feature together with
Nydus [2] container image service.
[1] https://sched.co/pcdL
[2] https://github.com/dragonflyoss/image-service
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
fuse/main.c | 2 +-
include/erofs/blobchunk.h | 3 +-
include/erofs/cache.h | 5 +++
include/erofs/config.h | 1 +
include/erofs/internal.h | 5 ++-
lib/blobchunk.c | 70 +++++++++++++++++++++++++++++++++------
man/mkfs.erofs.1 | 3 ++
mkfs/main.c | 19 ++++++++++-
8 files changed, 93 insertions(+), 15 deletions(-)
diff --git a/fuse/main.c b/fuse/main.c
index a92f06882b75..255965e30969 100644
--- a/fuse/main.c
+++ b/fuse/main.c
@@ -154,7 +154,7 @@ static int optional_opt_func(void *data, const char *arg, int key,
switch (key) {
case 1:
- ret = blob_open_ro(arg);
+ ret = blob_open_ro(arg + sizeof("--device=") - 1);
if (ret)
return -1;
++sbi.extra_devices;
diff --git a/include/erofs/blobchunk.h b/include/erofs/blobchunk.h
index b418227e0ef8..59a47013017f 100644
--- a/include/erofs/blobchunk.h
+++ b/include/erofs/blobchunk.h
@@ -13,6 +13,7 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
int erofs_blob_write_chunked_file(struct erofs_inode *inode);
int erofs_blob_remap(void);
void erofs_blob_exit(void);
-int erofs_blob_init(void);
+int erofs_blob_init(const char *blobfile_path);
+int erofs_generate_devtable(void);
#endif
diff --git a/include/erofs/cache.h b/include/erofs/cache.h
index e324d929b0b9..b19d54e1b4f4 100644
--- a/include/erofs/cache.h
+++ b/include/erofs/cache.h
@@ -19,6 +19,8 @@ struct erofs_buffer_block;
#define INODE 2
/* shared xattrs */
#define XATTR 3
+/* device table */
+#define DEVT 4
struct erofs_bhops {
bool (*preflush)(struct erofs_buffer_head *bh);
@@ -56,6 +58,9 @@ static inline const int get_alignsize(int type, int *type_ret)
} else if (type == XATTR) {
*type_ret = META;
return sizeof(struct erofs_xattr_entry);
+ } else if (type == DEVT) {
+ *type_ret = META;
+ return EROFS_DEVT_SLOT_SIZE;
}
if (type == META)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index a18c88301279..8d459c692dac 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -51,6 +51,7 @@ struct erofs_configure {
/* related arguments for mkfs.erofs */
char *c_img_path;
char *c_src_path;
+ char *c_blobdev_path;
char *c_compress_hints_file;
char *c_compr_alg_master;
int c_compr_level_master;
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 974c069baa4f..f22a016373ca 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -93,7 +93,10 @@ struct erofs_sb_info {
u32 checksum;
u16 extra_devices;
- u16 device_id_mask;
+ union {
+ u16 devt_slotoff; /* used for mkfs */
+ u16 device_id_mask; /* used for others */
+ };
};
/* global sbi */
diff --git a/lib/blobchunk.c b/lib/blobchunk.c
index 08e5cfb287f2..a10ca8cc8750 100644
--- a/lib/blobchunk.c
+++ b/lib/blobchunk.c
@@ -25,6 +25,8 @@ struct erofs_blobchunk {
static struct hashmap blob_hashmap;
static FILE *blobfile;
static erofs_blk_t remapped_base;
+static bool multidev;
+static struct erofs_buffer_head *bh_devt;
static struct erofs_blobchunk *erofs_blob_getchunk(int fd,
unsigned int chunksize)
@@ -103,22 +105,28 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode,
{
struct erofs_inode_chunk_index idx = {0};
erofs_blk_t extent_start = EROFS_NULL_ADDR;
- erofs_blk_t extent_end = EROFS_NULL_ADDR;
- unsigned int dst, src, unit, num_extents;
+ erofs_blk_t extent_end, extents_blks;
+ unsigned int dst, src, unit;
bool first_extent = true;
+ erofs_blk_t base_blkaddr = 0;
if (inode->u.chunkformat & EROFS_CHUNK_FORMAT_INDEXES)
unit = sizeof(struct erofs_inode_chunk_index);
else
unit = EROFS_BLOCK_MAP_ENTRY_SIZE;
+ if (multidev)
+ idx.device_id = 1;
+ else
+ base_blkaddr = remapped_base;
+
for (dst = src = 0; dst < inode->extent_isize;
src += sizeof(void *), dst += unit) {
struct erofs_blobchunk *chunk;
chunk = *(void **)(inode->chunkindexes + src);
- idx.blkaddr = chunk->blkaddr + remapped_base;
+ idx.blkaddr = base_blkaddr + chunk->blkaddr;
if (extent_start != EROFS_NULL_ADDR &&
idx.blkaddr == extent_end + 1) {
extent_end = idx.blkaddr;
@@ -141,11 +149,11 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode,
off = roundup(off, unit);
if (extent_start == EROFS_NULL_ADDR)
- num_extents = 0;
+ extents_blks = 0;
else
- num_extents = (extent_end - extent_start) + 1;
- erofs_droid_blocklist_write_extent(inode, extent_start, num_extents,
- first_extent, true);
+ extents_blks = (extent_end - extent_start) + 1;
+ erofs_droid_blocklist_write_extent(inode, extent_start, extents_blks,
+ first_extent, true);
return dev_write(inode->chunkindexes, off, inode->extent_isize);
}
@@ -208,6 +216,20 @@ int erofs_blob_remap(void)
fflush(blobfile);
length = ftell(blobfile);
+ if (multidev) {
+ struct erofs_deviceslot dis = {
+ .blocks = erofs_blknr(length),
+ };
+
+ pos_out = erofs_btell(bh_devt, false);
+ ret = dev_write(&dis, pos_out, sizeof(dis));
+ if (ret)
+ return ret;
+
+ bh_devt->op = &erofs_drop_directly_bhops;
+ erofs_bdrop(bh_devt, false);
+ return 0;
+ }
bh = erofs_balloc(DATA, length, 0, 0);
if (IS_ERR(bh))
return PTR_ERR(bh);
@@ -231,16 +253,42 @@ void erofs_blob_exit(void)
hashmap_free(&blob_hashmap, 1);
}
-int erofs_blob_init(void)
+int erofs_blob_init(const char *blobfile_path)
{
+ if (!blobfile_path) {
#ifdef HAVE_TMPFILE64
- blobfile = tmpfile64();
+ blobfile = tmpfile64();
#else
- blobfile = tmpfile();
+ blobfile = tmpfile();
#endif
+ multidev = false;
+ } else {
+ blobfile = fopen(blobfile_path, "wb");
+ multidev = true;
+ }
if (!blobfile)
- return -ENOMEM;
+ return -EACCES;
hashmap_init(&blob_hashmap, erofs_blob_hashmap_cmp, 0);
return 0;
}
+
+int erofs_generate_devtable(void)
+{
+ struct erofs_deviceslot dis;
+
+ if (!multidev)
+ return 0;
+
+ bh_devt = erofs_balloc(DEVT, sizeof(dis), 0, 0);
+ if (IS_ERR(bh_devt))
+ return PTR_ERR(bh_devt);
+
+ dis = (struct erofs_deviceslot) {};
+ erofs_mapbh(bh_devt->block);
+ bh_devt->op = &erofs_skip_write_bhops;
+ sbi.devt_slotoff = erofs_btell(bh_devt, false) / EROFS_DEVT_SLOT_SIZE;
+ sbi.extra_devices = 1;
+ erofs_sb_set_device_table();
+ return 0;
+}
diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1
index c7829c3f1c8f..71a26d88121a 100644
--- a/man/mkfs.erofs.1
+++ b/man/mkfs.erofs.1
@@ -66,6 +66,9 @@ like this: "c1b9d5a2-f162-11cf-9ece-0020afc76f16".
.B \-\-all-root
Make all files owned by root.
.TP
+.BI "\-\-blobdev " file
+Specify another extra blob device to store chunk-based data.
+.TP
.BI "\-\-chunksize " #
Generate chunk-based files with #-byte chunks.
.TP
diff --git a/mkfs/main.c b/mkfs/main.c
index 2604bf2abd6b..29042c801794 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -47,6 +47,7 @@ static struct option long_options[] = {
{"compress-hints", required_argument, NULL, 10},
{"chunksize", required_argument, NULL, 11},
{"quiet", no_argument, 0, 12},
+ {"blobdev", required_argument, NULL, 13},
#ifdef WITH_ANDROID
{"mount-point", required_argument, NULL, 512},
{"product-out", required_argument, NULL, 513},
@@ -83,6 +84,7 @@ static void usage(void)
" -UX use a given filesystem UUID\n"
#endif
" --all-root make all files owned by root\n"
+ " --blobdev=X specify an extra device X to store chunked data\n"
" --chunksize=# generate chunk-based files with #-byte chunks\n"
" --compress-hints=X specify a file to configure per-file compression strategy\n"
" --exclude-path=X avoid including file X (X = exact literal path)\n"
@@ -348,6 +350,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
case 12:
quiet = true;
break;
+ case 13:
+ cfg.c_blobdev_path = optarg;
+ break;
case 1:
usage();
exit(0);
@@ -360,6 +365,10 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
if (optind >= argc)
return -EINVAL;
+ if (cfg.c_blobdev_path && cfg.c_chunkbits < LOG_BLOCK_SIZE) {
+ erofs_err("--blobdev must be used together with --chunksize");
+ return -EINVAL;
+ }
cfg.c_img_path = strdup(argv[optind++]);
if (!cfg.c_img_path)
return -ENOMEM;
@@ -401,6 +410,8 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
.feature_incompat = cpu_to_le32(sbi.feature_incompat),
.feature_compat = cpu_to_le32(sbi.feature_compat &
~EROFS_FEATURE_COMPAT_SB_CHKSUM),
+ .extra_devices = cpu_to_le16(sbi.extra_devices),
+ .devt_slotoff = cpu_to_le16(sbi.devt_slotoff),
};
const unsigned int sb_blksize =
round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
@@ -549,7 +560,7 @@ int main(int argc, char **argv)
}
if (cfg.c_chunkbits) {
- err = erofs_blob_init();
+ err = erofs_blob_init(cfg.c_blobdev_path);
if (err)
return 1;
}
@@ -626,6 +637,12 @@ int main(int argc, char **argv)
goto exit;
}
+ err = erofs_generate_devtable();
+ if (err) {
+ erofs_err("Failed to generate device table: %s",
+ erofs_strerror(err));
+ goto exit;
+ }
#ifdef HAVE_LIBUUID
uuid_unparse_lower(sbi.uuid, uuid_str);
#endif
--
2.24.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v4 4/6] erofs-utils: dump: support multiple devices
2021-11-16 9:49 [PATCH v4 0/6] erofs-utils: add multiple device support Gao Xiang
` (2 preceding siblings ...)
2021-11-16 9:49 ` [PATCH v4 3/6] erofs-utils: mkfs: add extra blob " Gao Xiang
@ 2021-11-16 9:49 ` Gao Xiang
2021-11-16 9:49 ` [PATCH v4 5/6] erofs-utils: fsck: " Gao Xiang
2021-11-16 9:49 ` [PATCH v4 6/6] erofs-utils: get compression algorithms directly on mapping Gao Xiang
5 siblings, 0 replies; 9+ messages in thread
From: Gao Xiang @ 2021-11-16 9:49 UTC (permalink / raw)
To: linux-erofs
Cc: Yan Song, Peng Tao, Joseph Qi, Liu Bo, Changwei Ge, Gao Xiang, Liu Jiang
Add preliminary multiple device support for dump feature.
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
dump/main.c | 55 ++++++++++++++++++++++++++++++----------
include/erofs/internal.h | 1 +
man/dump.erofs.1 | 25 +++++-------------
3 files changed, 49 insertions(+), 32 deletions(-)
diff --git a/dump/main.c b/dump/main.c
index d0efe9505317..b7560eca1080 100644
--- a/dump/main.c
+++ b/dump/main.c
@@ -70,6 +70,7 @@ static struct erofs_statistics stats;
static struct option long_options[] = {
{"help", no_argument, NULL, 1},
{"nid", required_argument, NULL, 2},
+ {"device", required_argument, NULL, 3},
{0, 0, 0, 0},
};
@@ -84,6 +85,7 @@ static struct erofsdump_feature feature_lists[] = {
{ false, EROFS_FEATURE_INCOMPAT_LZ4_0PADDING, "0padding" },
{ false, EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER, "big_pcluster" },
{ false, EROFS_FEATURE_INCOMPAT_CHUNKED_FILE, "chunked_file" },
+ { false, EROFS_FEATURE_INCOMPAT_DEVICE_TABLE, "device_table" },
};
static int erofs_read_dir(erofs_nid_t nid, erofs_nid_t parent_nid);
@@ -95,12 +97,13 @@ static void usage(void)
{
fputs("usage: [options] IMAGE\n\n"
"Dump erofs layout from IMAGE, and [options] are:\n"
- " -S show statistic information of the image\n"
- " -V print the version number of dump.erofs and exit.\n"
- " -e show extent info (--nid is required)\n"
- " -s show information about superblock\n"
- " --nid=# show the target inode info of nid #\n"
- " --help display this help and exit.\n",
+ " -S show statistic information of the image\n"
+ " -V print the version number of dump.erofs and exit.\n"
+ " -e show extent info (--nid is required)\n"
+ " -s show information about superblock\n"
+ " --device=X specify an extra device to be used together\n"
+ " --nid=# show the target inode info of nid #\n"
+ " --help display this help and exit.\n",
stderr);
}
@@ -111,7 +114,7 @@ static void erofsdump_print_version(void)
static int erofsdump_parse_options_cfg(int argc, char **argv)
{
- int opt;
+ int opt, err;
while ((opt = getopt_long(argc, argv, "SVes",
long_options, NULL)) != -1) {
@@ -139,6 +142,12 @@ static int erofsdump_parse_options_cfg(int argc, char **argv)
case 1:
usage();
exit(0);
+ case 3:
+ err = blob_open_ro(optarg);
+ if (err)
+ return err;
+ ++sbi.extra_devices;
+ break;
default:
return -EINVAL;
}
@@ -423,6 +432,10 @@ static int erofsdump_map_blocks(struct erofs_inode *inode,
static void erofsdump_show_fileinfo(bool show_extent)
{
+ const char *ext_fmt[] = {
+ "%4d: %8" PRIu64 "..%8" PRIu64 " | %7" PRIu64 " : %10" PRIu64 "..%10" PRIu64 " | %7" PRIu64 "\n",
+ "%4d: %8" PRIu64 "..%8" PRIu64 " | %7" PRIu64 " : %10" PRIu64 "..%10" PRIu64 " | %7" PRIu64 " # device %u\n"
+ };
int err, i;
erofs_off_t size;
u16 access_mode;
@@ -482,16 +495,29 @@ static void erofsdump_show_fileinfo(bool show_extent)
fprintf(stdout, "\n Ext: logical offset | length : physical offset | length \n");
while (map.m_la < inode.i_size) {
+ struct erofs_map_dev mdev;
+
err = erofsdump_map_blocks(&inode, &map,
EROFS_GET_BLOCKS_FIEMAP);
if (err) {
- erofs_err("get file blocks range failed");
+ erofs_err("failed to get file blocks range");
return;
}
- fprintf(stdout, "%4d: %8" PRIu64 "..%8" PRIu64 " | %7" PRIu64 " : %10" PRIu64 "..%10" PRIu64 " | %7" PRIu64 "\n",
- extent_count++, map.m_la, map.m_la + map.m_llen, map.m_llen,
- map.m_pa, map.m_pa + map.m_plen, map.m_plen);
+ mdev = (struct erofs_map_dev) {
+ .m_deviceid = map.m_deviceid,
+ .m_pa = map.m_pa,
+ };
+ err = erofs_map_dev(&sbi, &mdev);
+ if (err) {
+ erofs_err("failed to map device");
+ return;
+ }
+
+ fprintf(stdout, ext_fmt[!!mdev.m_deviceid], extent_count++,
+ map.m_la, map.m_la + map.m_llen, map.m_llen,
+ mdev.m_pa, mdev.m_pa + map.m_plen, map.m_plen,
+ mdev.m_deviceid);
map.m_la += map.m_llen;
}
fprintf(stdout, "%s: %d extents found\n", path, extent_count);
@@ -658,7 +684,7 @@ int main(int argc, char **argv)
err = erofs_read_superblock();
if (err) {
erofs_err("failed to read superblock");
- goto exit;
+ goto exit_dev_close;
}
if (!dumpcfg.totalshow) {
@@ -673,13 +699,16 @@ int main(int argc, char **argv)
if (dumpcfg.show_extent && !dumpcfg.show_inode) {
usage();
- goto exit;
+ goto exit_dev_close;
}
if (dumpcfg.show_inode)
erofsdump_show_fileinfo(dumpcfg.show_extent);
+exit_dev_close:
+ dev_close();
exit:
+ blob_closeall();
erofs_exit_configure();
return err;
}
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index f22a016373ca..93e05bbc8271 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -300,6 +300,7 @@ int erofs_pread(struct erofs_inode *inode, char *buf,
erofs_off_t count, erofs_off_t offset);
int erofs_map_blocks(struct erofs_inode *inode,
struct erofs_map_blocks *map, int flags);
+int erofs_map_dev(struct erofs_sb_info *sbi, struct erofs_map_dev *map);
/* zmap.c */
int z_erofs_fill_inode(struct erofs_inode *vi);
int z_erofs_map_blocks_iter(struct erofs_inode *vi,
diff --git a/man/dump.erofs.1 b/man/dump.erofs.1
index 8233c89cdeb0..8efb161b65f1 100644
--- a/man/dump.erofs.1
+++ b/man/dump.erofs.1
@@ -5,24 +5,7 @@
dump.erofs \- retrieve directory and file entries, show specific file
or overall disk statistics information from an EROFS-formatted image.
.SH SYNOPSIS
-.B dump.erofs
-[
-.B \--nid
-.I inode NID
-]
-[
-.B \-e
-]
-[
-.B \-s
-]
-[
-.B \-S
-]
-[
-.B \-V
-]
-.I IMAGE
+\fBdump.erofs\fR [\fIOPTIONS\fR] \fIIMAGE\fR
.SH DESCRIPTION
.B dump.erofs
is used to retrieve erofs metadata from \fIIMAGE\fP and demonstrate
@@ -32,7 +15,11 @@ is used to retrieve erofs metadata from \fIIMAGE\fP and demonstrate
4) file extent information of the given inode NID.
.SH OPTIONS
.TP
-.BI \--nid " inode NID"
+.BI "\-\-device=" path
+Specify an extra device to be used together.
+You may give multiple `--device' options in the correct order.
+.TP
+.BI "\-\-nid=" NID
Specify an inode NID in order to print its file information.
.TP
.BI \-e
--
2.24.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v4 5/6] erofs-utils: fsck: support multiple devices
2021-11-16 9:49 [PATCH v4 0/6] erofs-utils: add multiple device support Gao Xiang
` (3 preceding siblings ...)
2021-11-16 9:49 ` [PATCH v4 4/6] erofs-utils: dump: support multiple devices Gao Xiang
@ 2021-11-16 9:49 ` Gao Xiang
2021-11-16 9:49 ` [PATCH v4 6/6] erofs-utils: get compression algorithms directly on mapping Gao Xiang
5 siblings, 0 replies; 9+ messages in thread
From: Gao Xiang @ 2021-11-16 9:49 UTC (permalink / raw)
To: linux-erofs
Cc: Yan Song, Peng Tao, Joseph Qi, Liu Bo, Changwei Ge, Gao Xiang, Liu Jiang
Add preliminary multiple device support for fsck feature.
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
fsck/main.c | 43 +++++++++++++++++++++++++++++++++----------
1 file changed, 33 insertions(+), 10 deletions(-)
diff --git a/fsck/main.c b/fsck/main.c
index b742e3579c59..7bee5605b9df 100644
--- a/fsck/main.c
+++ b/fsck/main.c
@@ -24,6 +24,7 @@ static struct erofsfsck_cfg fsckcfg;
static struct option long_options[] = {
{"help", no_argument, 0, 1},
{"extract", no_argument, 0, 2},
+ {"device", required_argument, 0, 3},
{0, 0, 0, 0},
};
@@ -34,6 +35,7 @@ static void usage(void)
" -V print the version number of fsck.erofs and exit.\n"
" -d# set output message level to # (maximum 9)\n"
" -p print total compression ratio of all files\n"
+ " --device=X specify an extra device to be used together\n"
" --extract check if all files are well encoded\n"
" --help display this help and exit.\n",
stderr);
@@ -46,7 +48,7 @@ static void erofsfsck_print_version(void)
static int erofsfsck_parse_options_cfg(int argc, char **argv)
{
- int opt, i;
+ int opt, ret;
while ((opt = getopt_long(argc, argv, "Vd:p",
long_options, NULL)) != -1) {
@@ -55,12 +57,12 @@ static int erofsfsck_parse_options_cfg(int argc, char **argv)
erofsfsck_print_version();
exit(0);
case 'd':
- i = atoi(optarg);
- if (i < EROFS_MSG_MIN || i > EROFS_MSG_MAX) {
- erofs_err("invalid debug level %d", i);
+ ret = atoi(optarg);
+ if (ret < EROFS_MSG_MIN || ret > EROFS_MSG_MAX) {
+ erofs_err("invalid debug level %d", ret);
return -EINVAL;
}
- cfg.c_dbg_lvl = i;
+ cfg.c_dbg_lvl = ret;
break;
case 'p':
fsckcfg.print_comp_ratio = true;
@@ -71,6 +73,12 @@ static int erofsfsck_parse_options_cfg(int argc, char **argv)
case 2:
fsckcfg.check_decomp = true;
break;
+ case 3:
+ ret = blob_open_ro(optarg);
+ if (ret)
+ return ret;
+ ++sbi.extra_devices;
+ break;
default:
return -EINVAL;
}
@@ -275,6 +283,7 @@ static int verify_compressed_inode(struct erofs_inode *inode)
struct erofs_map_blocks map = {
.index = UINT_MAX,
};
+ struct erofs_map_dev mdev;
int ret = 0;
u64 pchunk_len = 0;
erofs_off_t end = inode->i_size;
@@ -317,10 +326,21 @@ static int verify_compressed_inode(struct erofs_inode *inode)
BUG_ON(!buffer);
}
- ret = dev_read(0, raw, map.m_pa, map.m_plen);
+ mdev = (struct erofs_map_dev) {
+ .m_deviceid = map.m_deviceid,
+ .m_pa = map.m_pa,
+ };
+ ret = erofs_map_dev(&sbi, &mdev);
+ if (ret) {
+ erofs_err("failed to map device of m_pa %" PRIu64 ", m_deviceid %u @ nid %llu: %d",
+ map.m_pa, map.m_deviceid, inode->nid | 0ULL, ret);
+ goto out;
+ }
+
+ ret = dev_read(mdev.m_deviceid, raw, mdev.m_pa, map.m_plen);
if (ret < 0) {
erofs_err("failed to read compressed data of m_pa %" PRIu64 ", m_plen %" PRIu64 " @ nid %llu: %d",
- map.m_pa, map.m_plen, inode->nid | 0ULL, ret);
+ mdev.m_pa, map.m_plen, inode->nid | 0ULL, ret);
goto out;
}
@@ -336,7 +356,7 @@ static int verify_compressed_inode(struct erofs_inode *inode)
if (ret < 0) {
erofs_err("failed to decompress data of m_pa %" PRIu64 ", m_plen %" PRIu64 " @ nid %llu: %d",
- map.m_pa, map.m_plen, inode->nid | 0ULL, ret);
+ mdev.m_pa, map.m_plen, inode->nid | 0ULL, ret);
goto out;
}
}
@@ -558,12 +578,12 @@ int main(int argc, char **argv)
err = erofs_read_superblock();
if (err) {
erofs_err("failed to read superblock");
- goto exit;
+ goto exit_dev_close;
}
if (erofs_sb_has_sb_chksum() && erofs_check_sb_chksum()) {
erofs_err("failed to verify superblock checksum");
- goto exit;
+ goto exit_dev_close;
}
erofs_check_inode(sbi.root_nid, sbi.root_nid);
@@ -582,7 +602,10 @@ int main(int argc, char **argv)
}
}
+exit_dev_close:
+ dev_close();
exit:
+ blob_closeall();
erofs_exit_configure();
return err ? 1 : 0;
}
--
2.24.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH v4 6/6] erofs-utils: get compression algorithms directly on mapping
2021-11-16 9:49 [PATCH v4 0/6] erofs-utils: add multiple device support Gao Xiang
` (4 preceding siblings ...)
2021-11-16 9:49 ` [PATCH v4 5/6] erofs-utils: fsck: " Gao Xiang
@ 2021-11-16 9:49 ` Gao Xiang
2021-11-18 5:51 ` Yue Hu
5 siblings, 1 reply; 9+ messages in thread
From: Gao Xiang @ 2021-11-16 9:49 UTC (permalink / raw)
To: linux-erofs
Cc: Yan Song, Peng Tao, Joseph Qi, Liu Bo, Changwei Ge, Gao Xiang, Liu Jiang
Keep in sync with the latest kernel
commit 8f89926290c4 ("erofs: get compression algorithms directly on mapping")
And it also fixes fsck MicroLZMA support, btw.
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
fsck/main.c | 8 ++------
include/erofs/decompress.h | 5 -----
include/erofs/internal.h | 12 +++++++++---
lib/data.c | 10 ++--------
lib/zmap.c | 19 ++++++++++---------
5 files changed, 23 insertions(+), 31 deletions(-)
diff --git a/fsck/main.c b/fsck/main.c
index 7bee5605b9df..aefa881f740a 100644
--- a/fsck/main.c
+++ b/fsck/main.c
@@ -287,7 +287,7 @@ static int verify_compressed_inode(struct erofs_inode *inode)
int ret = 0;
u64 pchunk_len = 0;
erofs_off_t end = inode->i_size;
- unsigned int algorithmformat, raw_size = 0, buffer_size = 0;
+ unsigned int raw_size = 0, buffer_size = 0;
char *raw = NULL, *buffer = NULL;
while (end > 0) {
@@ -310,10 +310,6 @@ static int verify_compressed_inode(struct erofs_inode *inode)
if (!fsckcfg.check_decomp || !(map.m_flags & EROFS_MAP_MAPPED))
continue;
- algorithmformat = map.m_flags & EROFS_MAP_ZIPPED ?
- Z_EROFS_COMPRESSION_LZ4 :
- Z_EROFS_COMPRESSION_SHIFTED;
-
if (map.m_plen > raw_size) {
raw_size = map.m_plen;
raw = realloc(raw, raw_size);
@@ -350,7 +346,7 @@ static int verify_compressed_inode(struct erofs_inode *inode)
.decodedskip = 0,
.inputsize = map.m_plen,
.decodedlength = map.m_llen,
- .alg = algorithmformat,
+ .alg = map.m_algorithmformat,
.partial_decoding = 0
});
diff --git a/include/erofs/decompress.h b/include/erofs/decompress.h
index 0ba2b08daa73..3d0d9633865d 100644
--- a/include/erofs/decompress.h
+++ b/include/erofs/decompress.h
@@ -8,11 +8,6 @@
#include "internal.h"
-enum {
- Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX,
- Z_EROFS_COMPRESSION_RUNTIME_MAX
-};
-
struct z_erofs_decompress_req {
char *in, *out;
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 93e05bbc8271..666d1f2df466 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -252,7 +252,7 @@ static inline const char *erofs_strerror(int err)
enum {
BH_Meta,
BH_Mapped,
- BH_Zipped,
+ BH_Encoded,
BH_FullMapped,
};
@@ -260,8 +260,8 @@ enum {
#define EROFS_MAP_MAPPED (1 << BH_Mapped)
/* Located in metadata (could be copied from bd_inode) */
#define EROFS_MAP_META (1 << BH_Meta)
-/* The extent has been compressed */
-#define EROFS_MAP_ZIPPED (1 << BH_Zipped)
+/* The extent is encoded */
+#define EROFS_MAP_ENCODED (1 << BH_Encoded)
/* The length of extent is full */
#define EROFS_MAP_FULL_MAPPED (1 << BH_FullMapped)
@@ -272,6 +272,7 @@ struct erofs_map_blocks {
u64 m_plen, m_llen;
unsigned short m_deviceid;
+ char m_algorithmformat;
unsigned int m_flags;
erofs_blk_t index;
};
@@ -282,6 +283,11 @@ struct erofs_map_blocks {
*/
#define EROFS_GET_BLOCKS_FIEMAP 0x0002
+enum {
+ Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX,
+ Z_EROFS_COMPRESSION_RUNTIME_MAX
+};
+
struct erofs_map_dev {
erofs_off_t m_pa;
unsigned int m_deviceid;
diff --git a/lib/data.c b/lib/data.c
index 136c0d97ab45..27710f941615 100644
--- a/lib/data.c
+++ b/lib/data.c
@@ -226,12 +226,11 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,
};
struct erofs_map_dev mdev;
bool partial;
- unsigned int algorithmformat, bufsize;
+ unsigned int bufsize = 0;
char *raw = NULL;
int ret = 0;
end = offset + size;
- bufsize = 0;
while (end > offset) {
map.m_la = end - 1;
@@ -288,18 +287,13 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,
if (ret < 0)
break;
- if (map.m_flags & EROFS_MAP_ZIPPED)
- algorithmformat = inode->z_algorithmtype[0];
- else
- algorithmformat = Z_EROFS_COMPRESSION_SHIFTED;
-
ret = z_erofs_decompress(&(struct z_erofs_decompress_req) {
.in = raw,
.out = buffer + end - offset,
.decodedskip = skip,
.inputsize = map.m_plen,
.decodedlength = length,
- .alg = algorithmformat,
+ .alg = map.m_algorithmformat,
.partial_decoding = partial
});
if (ret < 0)
diff --git a/lib/zmap.c b/lib/zmap.c
index 9dd0c7633a45..3715c47e3647 100644
--- a/lib/zmap.c
+++ b/lib/zmap.c
@@ -72,7 +72,7 @@ struct z_erofs_maprecorder {
unsigned long lcn;
/* compression extent information gathered */
- u8 type;
+ u8 type, headtype;
u16 clusterofs;
u16 delta[2];
erofs_blk_t pblk, compressedlcs;
@@ -390,9 +390,8 @@ static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
}
return z_erofs_extent_lookback(m, m->delta[0]);
case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
- map->m_flags &= ~EROFS_MAP_ZIPPED;
- /* fallthrough */
case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
+ m->headtype = m->type;
map->m_la = (lcn << lclusterbits) | m->clusterofs;
break;
default:
@@ -415,7 +414,7 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN &&
m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD);
- if (!(map->m_flags & EROFS_MAP_ZIPPED) ||
+ if (m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN ||
!(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) {
map->m_plen = 1 << lclusterbits;
return 0;
@@ -548,15 +547,13 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi,
if (err)
goto out;
- map->m_flags = EROFS_MAP_ZIPPED; /* by default, compressed */
+ map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED;
end = (m.lcn + 1ULL) << lclusterbits;
switch (m.type) {
case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
- if (endoff >= m.clusterofs)
- map->m_flags &= ~EROFS_MAP_ZIPPED;
- /* fallthrough */
case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
if (endoff >= m.clusterofs) {
+ m.headtype = m.type;
map->m_la = (m.lcn << lclusterbits) | m.clusterofs;
break;
}
@@ -586,12 +583,16 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi,
map->m_llen = end - map->m_la;
map->m_pa = blknr_to_addr(m.pblk);
- map->m_flags |= EROFS_MAP_MAPPED;
err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
if (err)
goto out;
+ if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN)
+ map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED;
+ else
+ map->m_algorithmformat = vi->z_algorithmtype[0];
+
if (flags & EROFS_GET_BLOCKS_FIEMAP) {
err = z_erofs_get_extent_decompressedlen(&m);
if (!err)
--
2.24.4
^ permalink raw reply related [flat|nested] 9+ messages in thread* Re: [PATCH v4 6/6] erofs-utils: get compression algorithms directly on mapping
2021-11-16 9:49 ` [PATCH v4 6/6] erofs-utils: get compression algorithms directly on mapping Gao Xiang
@ 2021-11-18 5:51 ` Yue Hu
2021-11-18 6:01 ` Gao Xiang
0 siblings, 1 reply; 9+ messages in thread
From: Yue Hu @ 2021-11-18 5:51 UTC (permalink / raw)
To: Gao Xiang
Cc: Yan Song, Peng Tao, Joseph Qi, Liu Bo, Changwei Ge, Liu Jiang,
linux-erofs
On Tue, 16 Nov 2021 17:49:39 +0800
Gao Xiang <hsiangkao@linux.alibaba.com> wrote:
> Keep in sync with the latest kernel
> commit 8f89926290c4 ("erofs: get compression algorithms directly on mapping")
>
> And it also fixes fsck MicroLZMA support, btw.
>
> Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
> ---
> fsck/main.c | 8 ++------
> include/erofs/decompress.h | 5 -----
> include/erofs/internal.h | 12 +++++++++---
> lib/data.c | 10 ++--------
> lib/zmap.c | 19 ++++++++++---------
> 5 files changed, 23 insertions(+), 31 deletions(-)
>
> diff --git a/fsck/main.c b/fsck/main.c
> index 7bee5605b9df..aefa881f740a 100644
> --- a/fsck/main.c
> +++ b/fsck/main.c
> @@ -287,7 +287,7 @@ static int verify_compressed_inode(struct erofs_inode *inode)
> int ret = 0;
> u64 pchunk_len = 0;
> erofs_off_t end = inode->i_size;
> - unsigned int algorithmformat, raw_size = 0, buffer_size = 0;
> + unsigned int raw_size = 0, buffer_size = 0;
> char *raw = NULL, *buffer = NULL;
>
> while (end > 0) {
> @@ -310,10 +310,6 @@ static int verify_compressed_inode(struct erofs_inode *inode)
> if (!fsckcfg.check_decomp || !(map.m_flags & EROFS_MAP_MAPPED))
> continue;
>
> - algorithmformat = map.m_flags & EROFS_MAP_ZIPPED ?
> - Z_EROFS_COMPRESSION_LZ4 :
> - Z_EROFS_COMPRESSION_SHIFTED;
> -
> if (map.m_plen > raw_size) {
> raw_size = map.m_plen;
> raw = realloc(raw, raw_size);
> @@ -350,7 +346,7 @@ static int verify_compressed_inode(struct erofs_inode *inode)
> .decodedskip = 0,
> .inputsize = map.m_plen,
> .decodedlength = map.m_llen,
> - .alg = algorithmformat,
> + .alg = map.m_algorithmformat,
> .partial_decoding = 0
> });
>
> diff --git a/include/erofs/decompress.h b/include/erofs/decompress.h
> index 0ba2b08daa73..3d0d9633865d 100644
> --- a/include/erofs/decompress.h
> +++ b/include/erofs/decompress.h
> @@ -8,11 +8,6 @@
>
> #include "internal.h"
>
> -enum {
> - Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX,
> - Z_EROFS_COMPRESSION_RUNTIME_MAX
> -};
> -
> struct z_erofs_decompress_req {
> char *in, *out;
>
> diff --git a/include/erofs/internal.h b/include/erofs/internal.h
> index 93e05bbc8271..666d1f2df466 100644
> --- a/include/erofs/internal.h
> +++ b/include/erofs/internal.h
> @@ -252,7 +252,7 @@ static inline const char *erofs_strerror(int err)
> enum {
> BH_Meta,
> BH_Mapped,
> - BH_Zipped,
> + BH_Encoded,
> BH_FullMapped,
> };
>
> @@ -260,8 +260,8 @@ enum {
> #define EROFS_MAP_MAPPED (1 << BH_Mapped)
> /* Located in metadata (could be copied from bd_inode) */
> #define EROFS_MAP_META (1 << BH_Meta)
> -/* The extent has been compressed */
> -#define EROFS_MAP_ZIPPED (1 << BH_Zipped)
> +/* The extent is encoded */
> +#define EROFS_MAP_ENCODED (1 << BH_Encoded)
> /* The length of extent is full */
> #define EROFS_MAP_FULL_MAPPED (1 << BH_FullMapped)
>
> @@ -272,6 +272,7 @@ struct erofs_map_blocks {
> u64 m_plen, m_llen;
>
> unsigned short m_deviceid;
> + char m_algorithmformat;
> unsigned int m_flags;
> erofs_blk_t index;
> };
> @@ -282,6 +283,11 @@ struct erofs_map_blocks {
> */
> #define EROFS_GET_BLOCKS_FIEMAP 0x0002
>
> +enum {
> + Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX,
> + Z_EROFS_COMPRESSION_RUNTIME_MAX
> +};
> +
> struct erofs_map_dev {
> erofs_off_t m_pa;
> unsigned int m_deviceid;
> diff --git a/lib/data.c b/lib/data.c
> index 136c0d97ab45..27710f941615 100644
> --- a/lib/data.c
> +++ b/lib/data.c
> @@ -226,12 +226,11 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,
> };
> struct erofs_map_dev mdev;
> bool partial;
> - unsigned int algorithmformat, bufsize;
> + unsigned int bufsize = 0;
> char *raw = NULL;
> int ret = 0;
>
> end = offset + size;
> - bufsize = 0;
> while (end > offset) {
> map.m_la = end - 1;
>
> @@ -288,18 +287,13 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,
> if (ret < 0)
> break;
>
> - if (map.m_flags & EROFS_MAP_ZIPPED)
> - algorithmformat = inode->z_algorithmtype[0];
> - else
> - algorithmformat = Z_EROFS_COMPRESSION_SHIFTED;
> -
> ret = z_erofs_decompress(&(struct z_erofs_decompress_req) {
> .in = raw,
> .out = buffer + end - offset,
> .decodedskip = skip,
> .inputsize = map.m_plen,
> .decodedlength = length,
> - .alg = algorithmformat,
> + .alg = map.m_algorithmformat,
> .partial_decoding = partial
> });
> if (ret < 0)
> diff --git a/lib/zmap.c b/lib/zmap.c
> index 9dd0c7633a45..3715c47e3647 100644
> --- a/lib/zmap.c
> +++ b/lib/zmap.c
> @@ -72,7 +72,7 @@ struct z_erofs_maprecorder {
>
> unsigned long lcn;
> /* compression extent information gathered */
> - u8 type;
> + u8 type, headtype;
> u16 clusterofs;
> u16 delta[2];
> erofs_blk_t pblk, compressedlcs;
> @@ -390,9 +390,8 @@ static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
> }
> return z_erofs_extent_lookback(m, m->delta[0]);
> case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
> - map->m_flags &= ~EROFS_MAP_ZIPPED;
> - /* fallthrough */
> case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
> + m->headtype = m->type;
> map->m_la = (lcn << lclusterbits) | m->clusterofs;
> break;
> default:
> @@ -415,7 +414,7 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
>
> DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN &&
> m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD);
> - if (!(map->m_flags & EROFS_MAP_ZIPPED) ||
> + if (m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN ||
> !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) {
> map->m_plen = 1 << lclusterbits;
> return 0;
> @@ -548,15 +547,13 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi,
> if (err)
> goto out;
>
> - map->m_flags = EROFS_MAP_ZIPPED; /* by default, compressed */
> + map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED;
> end = (m.lcn + 1ULL) << lclusterbits;
> switch (m.type) {
> case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
> - if (endoff >= m.clusterofs)
> - map->m_flags &= ~EROFS_MAP_ZIPPED;
> - /* fallthrough */
> case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
> if (endoff >= m.clusterofs) {
> + m.headtype = m.type;
> map->m_la = (m.lcn << lclusterbits) | m.clusterofs;
> break;
> }
> @@ -586,12 +583,16 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi,
>
> map->m_llen = end - map->m_la;
> map->m_pa = blknr_to_addr(m.pblk);
> - map->m_flags |= EROFS_MAP_MAPPED;
>
> err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
> if (err)
> goto out;
>
> + if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN)
> + map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED;
> + else
> + map->m_algorithmformat = vi->z_algorithmtype[0];
> +
> if (flags & EROFS_GET_BLOCKS_FIEMAP) {
> err = z_erofs_get_extent_decompressedlen(&m);
> if (!err)
Reviewed-by: Yue Hu <huyue2@yulong.com>
Thanks.
^ permalink raw reply [flat|nested] 9+ messages in thread* Re: [PATCH v4 6/6] erofs-utils: get compression algorithms directly on mapping
2021-11-18 5:51 ` Yue Hu
@ 2021-11-18 6:01 ` Gao Xiang
0 siblings, 0 replies; 9+ messages in thread
From: Gao Xiang @ 2021-11-18 6:01 UTC (permalink / raw)
To: Yue Hu
Cc: Yan Song, Peng Tao, Joseph Qi, Liu Bo, Changwei Ge, Liu Jiang,
linux-erofs
On Thu, Nov 18, 2021 at 01:51:33PM +0800, Yue Hu wrote:
> On Tue, 16 Nov 2021 17:49:39 +0800
> Gao Xiang <hsiangkao@linux.alibaba.com> wrote:
...
> > map->m_pa = blknr_to_addr(m.pblk);
> > - map->m_flags |= EROFS_MAP_MAPPED;
> >
> > err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
> > if (err)
> > goto out;
> >
> > + if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN)
> > + map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED;
> > + else
> > + map->m_algorithmformat = vi->z_algorithmtype[0];
> > +
> > if (flags & EROFS_GET_BLOCKS_FIEMAP) {
> > err = z_erofs_get_extent_decompressedlen(&m);
> > if (!err)
>
> Reviewed-by: Yue Hu <huyue2@yulong.com>
>
Thanks for the review, will apply to dev branch later.
Thanks,
Gao Xiang
> Thanks.
^ permalink raw reply [flat|nested] 9+ messages in thread