* [PATCH v2 1/3] erofs-utils: add extra device I/O interface
@ 2021-11-12 12:31 Gao Xiang
2021-11-12 12:31 ` [PATCH v2 2/3] erofs-utils: fuse: add multiple device support Gao Xiang
2021-11-12 12:31 ` [PATCH v2 3/3] erofs-utils: add extra blob " Gao Xiang
0 siblings, 2 replies; 3+ messages in thread
From: Gao Xiang @ 2021-11-12 12:31 UTC (permalink / raw)
To: linux-erofs; +Cc: Gao Xiang
In order for erofsfuse to support multiple devices.
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
changes since v1:
- add erofsfuse multiple device support.
fsck/main.c | 8 ++++----
include/erofs/io.h | 10 ++++++----
lib/data.c | 6 +++---
lib/io.c | 48 +++++++++++++++++++++++++++++++++++++---------
lib/namei.c | 4 ++--
lib/super.c | 2 +-
lib/zmap.c | 4 ++--
mkfs/main.c | 2 +-
8 files changed, 58 insertions(+), 26 deletions(-)
diff --git a/fsck/main.c b/fsck/main.c
index d81d60024c8a..b742e3579c59 100644
--- a/fsck/main.c
+++ b/fsck/main.c
@@ -97,7 +97,7 @@ static int erofs_check_sb_chksum(void)
u32 crc;
struct erofs_super_block *sb;
- ret = blk_read(buf, 0, 1);
+ ret = blk_read(0, buf, 0, 1);
if (ret) {
erofs_err("failed to read superblock to check checksum: %d",
ret);
@@ -317,7 +317,7 @@ static int verify_compressed_inode(struct erofs_inode *inode)
BUG_ON(!buffer);
}
- ret = dev_read(raw, map.m_pa, map.m_plen);
+ ret = dev_read(0, raw, map.m_pa, map.m_plen);
if (ret < 0) {
erofs_err("failed to read compressed data of m_pa %" PRIu64 ", m_plen %" PRIu64 " @ nid %llu: %d",
map.m_pa, map.m_plen, inode->nid | 0ULL, ret);
@@ -381,7 +381,7 @@ static int erofs_verify_xattr(struct erofs_inode *inode)
}
addr = iloc(inode->nid) + inode->inode_isize;
- ret = dev_read(buf, addr, xattr_hdr_size);
+ ret = dev_read(0, buf, addr, xattr_hdr_size);
if (ret < 0) {
erofs_err("failed to read xattr header @ nid %llu: %d",
inode->nid | 0ULL, ret);
@@ -411,7 +411,7 @@ static int erofs_verify_xattr(struct erofs_inode *inode)
while (remaining > 0) {
unsigned int entry_sz;
- ret = dev_read(buf, addr, xattr_entry_size);
+ ret = dev_read(0, buf, addr, xattr_entry_size);
if (ret) {
erofs_err("failed to read xattr entry @ nid %llu: %d",
inode->nid | 0ULL, ret);
diff --git a/include/erofs/io.h b/include/erofs/io.h
index 9d73adc5f5f9..10a3681882e1 100644
--- a/include/erofs/io.h
+++ b/include/erofs/io.h
@@ -15,11 +15,13 @@
#define O_BINARY 0
#endif
+void blob_closeall(void);
+int blob_open_ro(const char *dev);
int dev_open(const char *devname);
int dev_open_ro(const char *dev);
void dev_close(void);
int dev_write(const void *buf, u64 offset, size_t len);
-int dev_read(void *buf, u64 offset, size_t len);
+int dev_read(int device_id, void *buf, u64 offset, size_t len);
int dev_fillzero(u64 offset, size_t len, bool padding);
int dev_fsync(void);
int dev_resize(erofs_blk_t nblocks);
@@ -38,10 +40,10 @@ static inline int blk_write(const void *buf, erofs_blk_t blkaddr,
blknr_to_addr(nblocks));
}
-static inline int blk_read(void *buf, erofs_blk_t start,
- u32 nblocks)
+static inline int blk_read(int device_id, void *buf,
+ erofs_blk_t start, u32 nblocks)
{
- return dev_read(buf, blknr_to_addr(start),
+ return dev_read(device_id, buf, blknr_to_addr(start),
blknr_to_addr(nblocks));
}
diff --git a/lib/data.c b/lib/data.c
index b5f0196c97dd..b83cbff3e731 100644
--- a/lib/data.c
+++ b/lib/data.c
@@ -91,7 +91,7 @@ int erofs_map_blocks(struct erofs_inode *inode,
pos = roundup(iloc(vi->nid) + vi->inode_isize +
vi->xattr_isize, unit) + unit * chunknr;
- err = blk_read(buf, erofs_blknr(pos), 1);
+ err = blk_read(0, buf, erofs_blknr(pos), 1);
if (err < 0)
return -EIO;
@@ -176,7 +176,7 @@ static int erofs_read_raw_data(struct erofs_inode *inode, char *buffer,
map.m_la = ptr;
}
- ret = dev_read(estart, map.m_pa, eend - map.m_la);
+ ret = dev_read(0, estart, map.m_pa, eend - map.m_la);
if (ret < 0)
return -EIO;
ptr = eend;
@@ -240,7 +240,7 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,
break;
}
}
- ret = dev_read(raw, map.m_pa, map.m_plen);
+ ret = dev_read(0, raw, map.m_pa, map.m_plen);
if (ret < 0)
break;
diff --git a/lib/io.c b/lib/io.c
index 279c7dd4b877..a0d366a4c3f1 100644
--- a/lib/io.c
+++ b/lib/io.c
@@ -26,6 +26,7 @@
static const char *erofs_devname;
int erofs_devfd = -1;
static u64 erofs_devsz;
+static unsigned int erofs_nblobs, erofs_blobfd[256];
int dev_get_blkdev_size(int fd, u64 *bytes)
{
@@ -106,6 +107,30 @@ int dev_open(const char *dev)
return 0;
}
+void blob_closeall(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < erofs_nblobs; ++i)
+ close(erofs_blobfd[i]);
+ erofs_nblobs = 0;
+}
+
+int blob_open_ro(const char *dev)
+{
+ int fd = open(dev, O_RDONLY | O_BINARY);
+
+ if (fd < 0) {
+ erofs_err("failed to open(%s).", dev);
+ return -errno;
+ }
+
+ erofs_blobfd[erofs_nblobs] = fd;
+ erofs_info("successfully to open blob%u %s", erofs_nblobs, dev);
+ ++erofs_nblobs;
+ return 0;
+}
+
/* XXX: temporary soluation. Disk I/O implementation needs to be refactored. */
int dev_open_ro(const char *dev)
{
@@ -229,9 +254,9 @@ int dev_resize(unsigned int blocks)
return dev_fillzero(st.st_size, length, true);
}
-int dev_read(void *buf, u64 offset, size_t len)
+int dev_read(int device_id, void *buf, u64 offset, size_t len)
{
- int ret;
+ int ret, fd;
if (cfg.c_dry_run)
return 0;
@@ -240,16 +265,21 @@ int dev_read(void *buf, u64 offset, size_t len)
erofs_err("buf is NULL");
return -EINVAL;
}
- if (offset >= erofs_devsz || len > erofs_devsz ||
- offset > erofs_devsz - len) {
- erofs_err("read posion[%" PRIu64 ", %zd] is too large beyond the end of device(%" PRIu64 ").",
- offset, len, erofs_devsz);
- return -EINVAL;
+
+ if (!device_id) {
+ fd = erofs_devfd;
+ } else {
+ if (device_id > erofs_nblobs) {
+ erofs_err("invalid device id %d", device_id);
+ return -ENODEV;
+ }
+ fd = erofs_blobfd[device_id - 1];
}
+
#ifdef HAVE_PREAD64
- ret = pread64(erofs_devfd, buf, len, (off64_t)offset);
+ ret = pread64(fd, buf, len, (off64_t)offset);
#else
- ret = pread(erofs_devfd, buf, len, (off_t)offset);
+ ret = pread(fd, buf, len, (off_t)offset);
#endif
if (ret != (int)len) {
erofs_err("Failed to read data from device - %s:[%" PRIu64 ", %zd].",
diff --git a/lib/namei.c b/lib/namei.c
index 56f199aba382..7377e74fad9b 100644
--- a/lib/namei.c
+++ b/lib/namei.c
@@ -30,7 +30,7 @@ int erofs_read_inode_from_disk(struct erofs_inode *vi)
struct erofs_inode_extended *die;
const erofs_off_t inode_loc = iloc(vi->nid);
- ret = dev_read(buf, inode_loc, sizeof(*dic));
+ ret = dev_read(0, buf, inode_loc, sizeof(*dic));
if (ret < 0)
return -EIO;
@@ -47,7 +47,7 @@ int erofs_read_inode_from_disk(struct erofs_inode *vi)
case EROFS_INODE_LAYOUT_EXTENDED:
vi->inode_isize = sizeof(struct erofs_inode_extended);
- ret = dev_read(buf + sizeof(*dic), inode_loc + sizeof(*dic),
+ ret = dev_read(0, buf + sizeof(*dic), inode_loc + sizeof(*dic),
sizeof(*die) - sizeof(*dic));
if (ret < 0)
return -EIO;
diff --git a/lib/super.c b/lib/super.c
index 0c3040394272..bf4d4318a321 100644
--- a/lib/super.c
+++ b/lib/super.c
@@ -30,7 +30,7 @@ int erofs_read_superblock(void)
unsigned int blkszbits;
int ret;
- ret = blk_read(data, 0, 1);
+ ret = blk_read(0, data, 0, 1);
if (ret < 0) {
erofs_err("cannot read erofs superblock: %d", ret);
return -EIO;
diff --git a/lib/zmap.c b/lib/zmap.c
index 7dbda87c3b13..9dd0c7633a45 100644
--- a/lib/zmap.c
+++ b/lib/zmap.c
@@ -38,7 +38,7 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode *vi)
vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY);
pos = round_up(iloc(vi->nid) + vi->inode_isize + vi->xattr_isize, 8);
- ret = dev_read(buf, pos, sizeof(buf));
+ ret = dev_read(0, buf, pos, sizeof(buf));
if (ret < 0)
return -EIO;
@@ -88,7 +88,7 @@ static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m,
if (map->index == eblk)
return 0;
- ret = blk_read(mpage, eblk, 1);
+ ret = blk_read(0, mpage, eblk, 1);
if (ret < 0)
return -EIO;
diff --git a/mkfs/main.c b/mkfs/main.c
index 4ea5467679b7..2604bf2abd6b 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -436,7 +436,7 @@ static int erofs_mkfs_superblock_csum_set(void)
u32 crc;
struct erofs_super_block *sb;
- ret = blk_read(buf, 0, 1);
+ ret = blk_read(0, buf, 0, 1);
if (ret) {
erofs_err("failed to read superblock to set checksum: %s",
erofs_strerror(ret));
--
2.24.4
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [PATCH v2 2/3] erofs-utils: fuse: add multiple device support
2021-11-12 12:31 [PATCH v2 1/3] erofs-utils: add extra device I/O interface Gao Xiang
@ 2021-11-12 12:31 ` Gao Xiang
2021-11-12 12:31 ` [PATCH v2 3/3] erofs-utils: add extra blob " Gao Xiang
1 sibling, 0 replies; 3+ messages in thread
From: Gao Xiang @ 2021-11-12 12:31 UTC (permalink / raw)
To: linux-erofs; +Cc: Gao Xiang
Keep in sync with the latest kernel
dfeab2e95a75 ("erofs: add multiple device support")
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
dump/main.c | 2 +-
fuse/main.c | 10 ++++++
include/erofs/defs.h | 32 +++++++++++++++++++
include/erofs/internal.h | 19 +++++++++++-
include/erofs_fs.h | 22 +++++++++++--
lib/data.c | 67 +++++++++++++++++++++++++++++++++-------
lib/super.c | 43 ++++++++++++++++++++++++--
7 files changed, 177 insertions(+), 18 deletions(-)
diff --git a/dump/main.c b/dump/main.c
index 401e6841aefb..912d98f70525 100644
--- a/dump/main.c
+++ b/dump/main.c
@@ -609,7 +609,7 @@ static void erofsdump_show_superblock(void)
fprintf(stdout, "Filesystem magic number: 0x%04X\n",
EROFS_SUPER_MAGIC_V1);
fprintf(stdout, "Filesystem blocks: %llu\n",
- sbi.blocks | 0ULL);
+ sbi.total_blocks | 0ULL);
fprintf(stdout, "Filesystem inode metadata start block: %u\n",
sbi.meta_blkaddr);
fprintf(stdout, "Filesystem shared xattr metadata start block: %u\n",
diff --git a/fuse/main.c b/fuse/main.c
index 8137421b1dea..21939cd15923 100644
--- a/fuse/main.c
+++ b/fuse/main.c
@@ -113,6 +113,7 @@ static struct options {
static const struct fuse_opt option_spec[] = {
OPTION("--dbglevel=%u", debug_lvl),
OPTION("--help", show_help),
+ FUSE_OPT_KEY("--device=", 1),
FUSE_OPT_END
};
@@ -148,7 +149,15 @@ static void erofsfuse_dumpcfg(void)
static int optional_opt_func(void *data, const char *arg, int key,
struct fuse_args *outargs)
{
+ int ret;
+
switch (key) {
+ case 1:
+ ret = blob_open_ro(arg);
+ if (ret)
+ return -1;
+ ++sbi.extra_devices;
+ return 0;
case FUSE_OPT_KEY_NONOPT:
if (fusecfg.mountpoint)
return -1; /* Too many args */
@@ -237,6 +246,7 @@ int main(int argc, char *argv[])
ret = fuse_main(args.argc, args.argv, &erofs_ops, NULL);
err_dev_close:
+ blob_closeall();
dev_close();
err_fuse_free_args:
fuse_opt_free_args(&args);
diff --git a/include/erofs/defs.h b/include/erofs/defs.h
index 96bbb6574ff3..5283f5d3c6f3 100644
--- a/include/erofs/defs.h
+++ b/include/erofs/defs.h
@@ -252,6 +252,38 @@ static inline u32 get_unaligned_le32(const u8 *p)
(n) & (1ULL << 1) ? 1 : 0 \
)
+static inline unsigned int fls_long(unsigned long x)
+{
+ return x ? sizeof(x) * 8 - __builtin_clz(x) : 0;
+}
+
+/**
+ * __roundup_pow_of_two() - round up to nearest power of two
+ * @n: value to round up
+ */
+static inline __attribute__((const))
+unsigned long __roundup_pow_of_two(unsigned long n)
+{
+ return 1UL << fls_long(n - 1);
+}
+
+/**
+ * roundup_pow_of_two - round the given value up to nearest power of two
+ * @n: parameter
+ *
+ * round the given value up to the nearest power of two
+ * - the result is undefined when n == 0
+ * - this can be used to initialise global variables from constant data
+ */
+#define roundup_pow_of_two(n) \
+( \
+ __builtin_constant_p(n) ? ( \
+ ((n) == 1) ? 1 : \
+ (1UL << (ilog2((n) - 1) + 1)) \
+ ) : \
+ __roundup_pow_of_two(n) \
+ )
+
#ifndef __always_inline
#define __always_inline inline
#endif
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index f84e6b4f125d..974c069baa4f 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -60,8 +60,16 @@ typedef u32 erofs_blk_t;
struct erofs_buffer_head;
+struct erofs_device_info {
+ u32 blocks;
+ u32 mapped_blkaddr;
+};
+
struct erofs_sb_info {
- u64 blocks;
+ struct erofs_device_info *devs;
+
+ u64 total_blocks;
+ u64 primarydevice_blocks;
erofs_blk_t meta_blkaddr;
erofs_blk_t xattr_blkaddr;
@@ -84,6 +92,8 @@ struct erofs_sb_info {
u16 lz4_max_distance;
u32 checksum;
+ u16 extra_devices;
+ u16 device_id_mask;
};
/* global sbi */
@@ -112,6 +122,7 @@ EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING)
EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS)
EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER)
EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)
+EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE)
EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
#define EROFS_I_EA_INITED (1 << 0)
@@ -257,6 +268,7 @@ struct erofs_map_blocks {
erofs_off_t m_pa, m_la;
u64 m_plen, m_llen;
+ unsigned short m_deviceid;
unsigned int m_flags;
erofs_blk_t index;
};
@@ -267,6 +279,11 @@ struct erofs_map_blocks {
*/
#define EROFS_GET_BLOCKS_FIEMAP 0x0002
+struct erofs_map_dev {
+ erofs_off_t m_pa;
+ unsigned int m_deviceid;
+};
+
/* super.c */
int erofs_read_superblock(void);
diff --git a/include/erofs_fs.h b/include/erofs_fs.h
index 4291970753a8..9a918775750c 100644
--- a/include/erofs_fs.h
+++ b/include/erofs_fs.h
@@ -22,14 +22,27 @@
#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002
#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002
#define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004
+#define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 0x00000008
#define EROFS_ALL_FEATURE_INCOMPAT \
(EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \
EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \
EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \
- EROFS_FEATURE_INCOMPAT_CHUNKED_FILE)
+ EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \
+ EROFS_FEATURE_INCOMPAT_DEVICE_TABLE)
#define EROFS_SB_EXTSLOT_SIZE 16
+struct erofs_deviceslot {
+ union {
+ u8 uuid[16]; /* used for device manager later */
+ u8 userdata[64]; /* digest(sha256), etc. */
+ } u;
+ __le32 blocks; /* total fs blocks of this device */
+ __le32 mapped_blkaddr; /* map starting at mapped_blkaddr */
+ u8 reserved[56];
+};
+#define EROFS_DEVT_SLOT_SIZE sizeof(struct erofs_deviceslot)
+
/* erofs on-disk super block (currently 128 bytes) */
struct erofs_super_block {
__le32 magic; /* file system magic number */
@@ -55,7 +68,9 @@ struct erofs_super_block {
/* customized sliding window size instead of 64k by default */
__le16 lz4_max_distance;
} __packed u1;
- __u8 reserved2[42];
+ __le16 extra_devices; /* # of devices besides the primary device */
+ __le16 devt_slotoff; /* startoff = devt_slotoff * devt_slotsize */
+ __u8 reserved2[38];
};
/*
@@ -239,7 +254,7 @@ static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e)
/* 8-byte inode chunk indexes */
struct erofs_inode_chunk_index {
__le16 advise; /* always 0, don't care for now */
- __le16 device_id; /* back-end storage id, always 0 for now */
+ __le16 device_id; /* back-end storage id (with bits masked) */
__le32 blkaddr; /* start block address of this inode chunk */
};
@@ -404,6 +419,7 @@ static inline void erofs_check_ondisk_layout_definitions(void)
/* keep in sync between 2 index structures for better extendibility */
BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_index) !=
sizeof(struct z_erofs_vle_decompressed_index));
+ BUILD_BUG_ON(sizeof(struct erofs_deviceslot) != 128);
BUILD_BUG_ON(BIT(Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) <
Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1);
diff --git a/lib/data.c b/lib/data.c
index b83cbff3e731..c90b66fca628 100644
--- a/lib/data.c
+++ b/lib/data.c
@@ -72,6 +72,7 @@ int erofs_map_blocks(struct erofs_inode *inode,
erofs_off_t pos;
int err = 0;
+ map->m_deviceid = 0;
if (map->m_la >= inode->i_size) {
/* leave out-of-bound access unmapped */
map->m_flags = 0;
@@ -118,14 +119,8 @@ int erofs_map_blocks(struct erofs_inode *inode,
map->m_flags = 0;
break;
default:
- /* only one device is supported for now */
- if (idx->device_id) {
- erofs_err("invalid device id %u @ %" PRIu64 " for nid %llu",
- le16_to_cpu(idx->device_id),
- chunknr, vi->nid | 0ULL);
- err = -EFSCORRUPTED;
- goto out;
- }
+ map->m_deviceid = le16_to_cpu(idx->device_id) &
+ sbi.device_id_mask;
map->m_pa = blknr_to_addr(le32_to_cpu(idx->blkaddr));
map->m_flags = EROFS_MAP_MAPPED;
break;
@@ -135,12 +130,42 @@ out:
return err;
}
+int erofs_map_dev(struct erofs_sb_info *sbi, struct erofs_map_dev *map)
+{
+ struct erofs_device_info *dif;
+ int id;
+
+ if (map->m_deviceid) {
+ if (sbi->extra_devices < map->m_deviceid)
+ return -ENODEV;
+ } else if (sbi->extra_devices) {
+ for (id = 0; id < sbi->extra_devices; ++id) {
+ erofs_off_t startoff, length;
+
+ dif = sbi->devs + id;
+
+ if (!dif->mapped_blkaddr)
+ continue;
+ startoff = blknr_to_addr(dif->mapped_blkaddr);
+ length = blknr_to_addr(dif->blocks);
+
+ if (map->m_pa >= startoff &&
+ map->m_pa < startoff + length) {
+ map->m_pa -= startoff;
+ break;
+ }
+ }
+ }
+ return 0;
+}
+
static int erofs_read_raw_data(struct erofs_inode *inode, char *buffer,
erofs_off_t size, erofs_off_t offset)
{
struct erofs_map_blocks map = {
.index = UINT_MAX,
};
+ struct erofs_map_dev mdev;
int ret;
erofs_off_t ptr = offset;
@@ -155,6 +180,14 @@ static int erofs_read_raw_data(struct erofs_inode *inode, char *buffer,
DBG_BUGON(map.m_plen != map.m_llen);
+ mdev = (struct erofs_map_dev) {
+ .m_deviceid = map.m_deviceid,
+ .m_pa = map.m_pa,
+ };
+ ret = erofs_map_dev(&sbi, &mdev);
+ if (ret)
+ return ret;
+
/* trim extent */
eend = min(offset + size, map.m_la + map.m_llen);
DBG_BUGON(ptr < map.m_la);
@@ -172,11 +205,12 @@ static int erofs_read_raw_data(struct erofs_inode *inode, char *buffer,
}
if (ptr > map.m_la) {
- map.m_pa += ptr - map.m_la;
+ mdev.m_pa += ptr - map.m_la;
map.m_la = ptr;
}
- ret = dev_read(0, estart, map.m_pa, eend - map.m_la);
+ ret = dev_read(mdev.m_deviceid, estart, mdev.m_pa,
+ eend - map.m_la);
if (ret < 0)
return -EIO;
ptr = eend;
@@ -191,6 +225,7 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,
struct erofs_map_blocks map = {
.index = UINT_MAX,
};
+ struct erofs_map_dev mdev;
bool partial;
unsigned int algorithmformat, bufsize;
char *raw = NULL;
@@ -205,6 +240,16 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,
if (ret)
break;
+ /* no device id here, thus it will always succeed */
+ mdev = (struct erofs_map_dev) {
+ .m_pa = map.m_pa,
+ };
+ ret = erofs_map_dev(&sbi, &mdev);
+ if (ret) {
+ DBG_BUGON(1);
+ break;
+ }
+
/*
* trim to the needed size if the returned extent is quite
* larger than requested, and set up partial flag as well.
@@ -240,7 +285,7 @@ static int z_erofs_read_data(struct erofs_inode *inode, char *buffer,
break;
}
}
- ret = dev_read(0, raw, map.m_pa, map.m_plen);
+ ret = dev_read(mdev.m_deviceid, raw, mdev.m_pa, map.m_plen);
if (ret < 0)
break;
diff --git a/lib/super.c b/lib/super.c
index bf4d4318a321..3ccc551f18cf 100644
--- a/lib/super.c
+++ b/lib/super.c
@@ -23,6 +23,45 @@ static bool check_layout_compatibility(struct erofs_sb_info *sbi,
return true;
}
+static int erofs_init_devices(struct erofs_sb_info *sbi,
+ struct erofs_super_block *dsb)
+{
+ unsigned int ondisk_extradevs, i;
+ erofs_off_t pos;
+
+ sbi->total_blocks = sbi->primarydevice_blocks;
+
+ if (!erofs_sb_has_device_table())
+ ondisk_extradevs = 0;
+ else
+ ondisk_extradevs = le16_to_cpu(dsb->extra_devices);
+
+ if (ondisk_extradevs != sbi->extra_devices) {
+ erofs_err("extra devices don't match (ondisk %u, given %u)",
+ ondisk_extradevs, sbi->extra_devices);
+ return -EINVAL;
+ }
+ if (!ondisk_extradevs)
+ return 0;
+
+ sbi->device_id_mask = roundup_pow_of_two(ondisk_extradevs + 1) - 1;
+ sbi->devs = calloc(ondisk_extradevs, sizeof(*sbi->devs));
+ pos = le16_to_cpu(dsb->devt_slotoff) * EROFS_DEVT_SLOT_SIZE;
+ for (i = 0; i < ondisk_extradevs; ++i) {
+ struct erofs_deviceslot dis;
+ int ret;
+
+ ret = dev_read(0, &dis, pos, sizeof(dis));
+ if (ret < 0)
+ return ret;
+
+ sbi->devs[i].mapped_blkaddr = dis.mapped_blkaddr;
+ sbi->total_blocks += dis.blocks;
+ pos += EROFS_DEVT_SLOT_SIZE;
+ }
+ return 0;
+}
+
int erofs_read_superblock(void)
{
char data[EROFS_BLKSIZ];
@@ -56,7 +95,7 @@ int erofs_read_superblock(void)
if (!check_layout_compatibility(&sbi, dsb))
return ret;
- sbi.blocks = le32_to_cpu(dsb->blocks);
+ sbi.primarydevice_blocks = le32_to_cpu(dsb->blocks);
sbi.meta_blkaddr = le32_to_cpu(dsb->meta_blkaddr);
sbi.xattr_blkaddr = le32_to_cpu(dsb->xattr_blkaddr);
sbi.islotbits = EROFS_ISLOTBITS;
@@ -68,5 +107,5 @@ int erofs_read_superblock(void)
sbi.build_time_nsec = le32_to_cpu(dsb->build_time_nsec);
memcpy(&sbi.uuid, dsb->uuid, sizeof(dsb->uuid));
- return 0;
+ return erofs_init_devices(&sbi, dsb);
}
--
2.24.4
^ permalink raw reply related [flat|nested] 3+ messages in thread
* [PATCH v2 3/3] erofs-utils: add extra blob device support
2021-11-12 12:31 [PATCH v2 1/3] erofs-utils: add extra device I/O interface Gao Xiang
2021-11-12 12:31 ` [PATCH v2 2/3] erofs-utils: fuse: add multiple device support Gao Xiang
@ 2021-11-12 12:31 ` Gao Xiang
1 sibling, 0 replies; 3+ messages in thread
From: Gao Xiang @ 2021-11-12 12:31 UTC (permalink / raw)
To: linux-erofs; +Cc: Gao Xiang
In this patch, blob data from chunked-based files is redirected to
another blob file.
In order to redirect that, "--blobdev" should be used to specify
the output blob file/device for all chunk-based files, e.g.
mkfs.erofs --blobdev blob.erofs --chunksize 4096 foo.erofs foo
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
fuse/main.c | 2 +-
include/erofs/blobchunk.h | 3 +-
include/erofs/cache.h | 5 ++++
include/erofs/config.h | 1 +
include/erofs/internal.h | 5 +++-
lib/blobchunk.c | 58 +++++++++++++++++++++++++++++++++++----
mkfs/main.c | 15 +++++++++-
7 files changed, 80 insertions(+), 9 deletions(-)
diff --git a/fuse/main.c b/fuse/main.c
index 21939cd15923..de494b79b8ad 100644
--- a/fuse/main.c
+++ b/fuse/main.c
@@ -153,7 +153,7 @@ static int optional_opt_func(void *data, const char *arg, int key,
switch (key) {
case 1:
- ret = blob_open_ro(arg);
+ ret = blob_open_ro(arg + sizeof("--device=") - 1);
if (ret)
return -1;
++sbi.extra_devices;
diff --git a/include/erofs/blobchunk.h b/include/erofs/blobchunk.h
index b418227e0ef8..85efe0f1687f 100644
--- a/include/erofs/blobchunk.h
+++ b/include/erofs/blobchunk.h
@@ -13,6 +13,7 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
int erofs_blob_write_chunked_file(struct erofs_inode *inode);
int erofs_blob_remap(void);
void erofs_blob_exit(void);
-int erofs_blob_init(void);
+int erofs_blob_init(const char *);
+int erofs_generate_devtable(void);
#endif
diff --git a/include/erofs/cache.h b/include/erofs/cache.h
index e324d929b0b9..b19d54e1b4f4 100644
--- a/include/erofs/cache.h
+++ b/include/erofs/cache.h
@@ -19,6 +19,8 @@ struct erofs_buffer_block;
#define INODE 2
/* shared xattrs */
#define XATTR 3
+/* device table */
+#define DEVT 4
struct erofs_bhops {
bool (*preflush)(struct erofs_buffer_head *bh);
@@ -56,6 +58,9 @@ static inline const int get_alignsize(int type, int *type_ret)
} else if (type == XATTR) {
*type_ret = META;
return sizeof(struct erofs_xattr_entry);
+ } else if (type == DEVT) {
+ *type_ret = META;
+ return EROFS_DEVT_SLOT_SIZE;
}
if (type == META)
diff --git a/include/erofs/config.h b/include/erofs/config.h
index a18c88301279..8d459c692dac 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -51,6 +51,7 @@ struct erofs_configure {
/* related arguments for mkfs.erofs */
char *c_img_path;
char *c_src_path;
+ char *c_blobdev_path;
char *c_compress_hints_file;
char *c_compr_alg_master;
int c_compr_level_master;
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 974c069baa4f..f22a016373ca 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -93,7 +93,10 @@ struct erofs_sb_info {
u32 checksum;
u16 extra_devices;
- u16 device_id_mask;
+ union {
+ u16 devt_slotoff; /* used for mkfs */
+ u16 device_id_mask; /* used for others */
+ };
};
/* global sbi */
diff --git a/lib/blobchunk.c b/lib/blobchunk.c
index a0ff79c748c6..96d1d49e92f9 100644
--- a/lib/blobchunk.c
+++ b/lib/blobchunk.c
@@ -24,6 +24,8 @@ struct erofs_blobchunk {
static struct hashmap blob_hashmap;
static FILE *blobfile;
static erofs_blk_t remapped_base;
+static bool multidev;
+static struct erofs_buffer_head *bh_devt;
static struct erofs_blobchunk *erofs_blob_getchunk(int fd,
unsigned int chunksize)
@@ -102,19 +104,25 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode,
{
struct erofs_inode_chunk_index idx = {0};
unsigned int dst, src, unit;
+ erofs_blk_t base_blkaddr = 0;
if (inode->u.chunkformat & EROFS_CHUNK_FORMAT_INDEXES)
unit = sizeof(struct erofs_inode_chunk_index);
else
unit = EROFS_BLOCK_MAP_ENTRY_SIZE;
+ if (multidev)
+ idx.device_id = 1;
+ else
+ base_blkaddr = remapped_base;
+
for (dst = src = 0; dst < inode->extent_isize;
src += sizeof(void *), dst += unit) {
struct erofs_blobchunk *chunk;
chunk = *(void **)(inode->chunkindexes + src);
- idx.blkaddr = chunk->blkaddr + remapped_base;
+ idx.blkaddr = base_blkaddr + chunk->blkaddr;
if (unit == EROFS_BLOCK_MAP_ENTRY_SIZE)
memcpy(inode->chunkindexes + dst, &idx.blkaddr, unit);
else
@@ -183,6 +191,20 @@ int erofs_blob_remap(void)
fflush(blobfile);
length = ftell(blobfile);
+ if (multidev) {
+ struct erofs_deviceslot dis = {
+ .blocks = erofs_blknr(length),
+ };
+
+ pos_out = erofs_btell(bh_devt, false);
+ ret = dev_write(&dis, pos_out, sizeof(dis));
+ if (ret)
+ return ret;
+
+ bh_devt->op = &erofs_drop_directly_bhops;
+ erofs_bdrop(bh_devt, false);
+ return 0;
+ }
bh = erofs_balloc(DATA, length, 0, 0);
if (IS_ERR(bh))
return PTR_ERR(bh);
@@ -206,16 +228,42 @@ void erofs_blob_exit(void)
hashmap_free(&blob_hashmap, 1);
}
-int erofs_blob_init(void)
+int erofs_blob_init(const char *blobfile_path)
{
+ if (!blobfile_path) {
#ifdef HAVE_TMPFILE64
- blobfile = tmpfile64();
+ blobfile = tmpfile64();
#else
- blobfile = tmpfile();
+ blobfile = tmpfile();
#endif
+ multidev = false;
+ } else {
+ blobfile = fopen(blobfile_path, "wb");
+ multidev = true;
+ }
if (!blobfile)
- return -ENOMEM;
+ return -EACCES;
hashmap_init(&blob_hashmap, erofs_blob_hashmap_cmp, 0);
return 0;
}
+
+int erofs_generate_devtable(void)
+{
+ struct erofs_deviceslot dis;
+
+ if (!multidev)
+ return 0;
+
+ bh_devt = erofs_balloc(DEVT, sizeof(dis), 0, 0);
+ if (IS_ERR(bh_devt))
+ return PTR_ERR(bh_devt);
+
+ dis = (struct erofs_deviceslot) {};
+ erofs_mapbh(bh_devt->block);
+ bh_devt->op = &erofs_skip_write_bhops;
+ sbi.devt_slotoff = erofs_btell(bh_devt, false) / EROFS_DEVT_SLOT_SIZE;
+ sbi.extra_devices = 1;
+ erofs_sb_set_device_table();
+ return 0;
+}
diff --git a/mkfs/main.c b/mkfs/main.c
index 2604bf2abd6b..740e67be6d13 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -47,6 +47,7 @@ static struct option long_options[] = {
{"compress-hints", required_argument, NULL, 10},
{"chunksize", required_argument, NULL, 11},
{"quiet", no_argument, 0, 12},
+ {"blobdev", required_argument, NULL, 13},
#ifdef WITH_ANDROID
{"mount-point", required_argument, NULL, 512},
{"product-out", required_argument, NULL, 513},
@@ -83,6 +84,7 @@ static void usage(void)
" -UX use a given filesystem UUID\n"
#endif
" --all-root make all files owned by root\n"
+ " --blobdev=X specify an extra device X for chunk data\n"
" --chunksize=# generate chunk-based files with #-byte chunks\n"
" --compress-hints=X specify a file to configure per-file compression strategy\n"
" --exclude-path=X avoid including file X (X = exact literal path)\n"
@@ -348,6 +350,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
case 12:
quiet = true;
break;
+ case 13:
+ cfg.c_blobdev_path = optarg;
+ break;
case 1:
usage();
exit(0);
@@ -401,6 +406,8 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh,
.feature_incompat = cpu_to_le32(sbi.feature_incompat),
.feature_compat = cpu_to_le32(sbi.feature_compat &
~EROFS_FEATURE_COMPAT_SB_CHKSUM),
+ .extra_devices = cpu_to_le16(sbi.extra_devices),
+ .devt_slotoff = cpu_to_le16(sbi.devt_slotoff),
};
const unsigned int sb_blksize =
round_up(EROFS_SUPER_END, EROFS_BLKSIZ);
@@ -549,7 +556,7 @@ int main(int argc, char **argv)
}
if (cfg.c_chunkbits) {
- err = erofs_blob_init();
+ err = erofs_blob_init(cfg.c_blobdev_path);
if (err)
return 1;
}
@@ -626,6 +633,12 @@ int main(int argc, char **argv)
goto exit;
}
+ err = erofs_generate_devtable();
+ if (err) {
+ erofs_err("Failed to generate device table: %s",
+ erofs_strerror(err));
+ goto exit;
+ }
#ifdef HAVE_LIBUUID
uuid_unparse_lower(sbi.uuid, uuid_str);
#endif
--
2.24.4
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2021-11-12 12:32 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-11-12 12:31 [PATCH v2 1/3] erofs-utils: add extra device I/O interface Gao Xiang
2021-11-12 12:31 ` [PATCH v2 2/3] erofs-utils: fuse: add multiple device support Gao Xiang
2021-11-12 12:31 ` [PATCH v2 3/3] erofs-utils: add extra blob " Gao Xiang
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).