* [PATCH 2/4] erofs-utils: introduce sha256
2021-08-18 7:03 [PATCH 1/4] erofs-utils: introduce hashmap from git source Gao Xiang
@ 2021-08-18 7:03 ` Gao Xiang
2021-08-18 7:03 ` [PATCH 3/4] erofs-utils: introduce copy_file_range Gao Xiang
2021-08-18 7:03 ` [PATCH 4/4] erofs-utils: support chunk-based uncompressed files Gao Xiang
2 siblings, 0 replies; 5+ messages in thread
From: Gao Xiang @ 2021-08-18 7:03 UTC (permalink / raw)
To: linux-erofs, Chao Yu, Liu Bo; +Cc: Joseph Qi, Eryu Guan, Liu Jiang, Peng Tao
A simple sha256 approach copied from e2fsprogs.
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
lib/Makefile.am | 2 +-
lib/sha256.c | 248 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 249 insertions(+), 1 deletion(-)
create mode 100644 lib/sha256.c
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 0768d6aa5871..8e4ed37f9c70 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -22,7 +22,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
noinst_HEADERS += compressor.h
liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
namei.c data.c compress.c compressor.c zmap.c decompress.c \
- hashmap.c
+ hashmap.c sha256.c
liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
if ENABLE_LZ4
liberofs_la_CFLAGS += ${LZ4_CFLAGS}
diff --git a/lib/sha256.c b/lib/sha256.c
new file mode 100644
index 000000000000..68e039ab9c91
--- /dev/null
+++ b/lib/sha256.c
@@ -0,0 +1,248 @@
+/*
+ * sha256.c --- The sha256 algorithm
+ *
+ * Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
+ * (copied from libtomcrypt and then relicensed under GPLv2)
+ *
+ * %Begin-Header%
+ * This file may be redistributed under the terms of the GNU Library
+ * General Public License, version 2.
+ * %End-Header%
+ */
+#include "erofs/defs.h"
+#include <string.h>
+
+static const __u32 K[64] = {
+ 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL,
+ 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL,
+ 0x243185beUL, 0x550c7dc3UL, 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL,
+ 0xc19bf174UL, 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL,
+ 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, 0x983e5152UL,
+ 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, 0xc6e00bf3UL, 0xd5a79147UL,
+ 0x06ca6351UL, 0x14292967UL, 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL,
+ 0x53380d13UL, 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL,
+ 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, 0xd192e819UL,
+ 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, 0x19a4c116UL, 0x1e376c08UL,
+ 0x2748774cUL, 0x34b0bcb5UL, 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL,
+ 0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL,
+ 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL
+};
+
+/* Various logical functions */
+#define Ch(x,y,z) (z ^ (x & (y ^ z)))
+#define Maj(x,y,z) (((x | y) & z) | (x & y))
+#define S(x, n) RORc((x),(n))
+#define R(x, n) (((x)&0xFFFFFFFFUL)>>(n))
+#define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22))
+#define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25))
+#define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3))
+#define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10))
+#define RORc(x, y) ( ((((__u32)(x)&0xFFFFFFFFUL)>>(__u32)((y)&31)) | ((__u32)(x)<<(__u32)(32-((y)&31)))) & 0xFFFFFFFFUL)
+
+#define RND(a,b,c,d,e,f,g,h,i) \
+ t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \
+ t1 = Sigma0(a) + Maj(a, b, c); \
+ d += t0; \
+ h = t0 + t1;
+
+#define STORE64H(x, y) \
+ do { \
+ (y)[0] = (unsigned char)(((x)>>56)&255);\
+ (y)[1] = (unsigned char)(((x)>>48)&255);\
+ (y)[2] = (unsigned char)(((x)>>40)&255);\
+ (y)[3] = (unsigned char)(((x)>>32)&255);\
+ (y)[4] = (unsigned char)(((x)>>24)&255);\
+ (y)[5] = (unsigned char)(((x)>>16)&255);\
+ (y)[6] = (unsigned char)(((x)>>8)&255);\
+ (y)[7] = (unsigned char)((x)&255); } while(0)
+
+#define STORE32H(x, y) \
+ do { (y)[0] = (unsigned char)(((x)>>24)&255); (y)[1] = (unsigned char)(((x)>>16)&255); \
+ (y)[2] = (unsigned char)(((x)>>8)&255); (y)[3] = (unsigned char)((x)&255); } while(0)
+
+#define LOAD32H(x, y) \
+ do { x = ((__u32)((y)[0] & 255)<<24) | \
+ ((__u32)((y)[1] & 255)<<16) | \
+ ((__u32)((y)[2] & 255)<<8) | \
+ ((__u32)((y)[3] & 255)); } while(0)
+
+struct sha256_state {
+ __u64 length;
+ __u32 state[8], curlen;
+ unsigned char buf[64];
+};
+
+/* This is a highly simplified version from libtomcrypt */
+struct hash_state {
+ struct sha256_state sha256;
+};
+
+static void sha256_compress(struct hash_state * md, const unsigned char *buf)
+{
+ __u32 S[8], W[64], t0, t1;
+ __u32 t;
+ int i;
+
+ /* copy state into S */
+ for (i = 0; i < 8; i++) {
+ S[i] = md->sha256.state[i];
+ }
+
+ /* copy the state into 512-bits into W[0..15] */
+ for (i = 0; i < 16; i++) {
+ LOAD32H(W[i], buf + (4*i));
+ }
+
+ /* fill W[16..63] */
+ for (i = 16; i < 64; i++) {
+ W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16];
+ }
+
+ /* Compress */
+ for (i = 0; i < 64; ++i) {
+ RND(S[0],S[1],S[2],S[3],S[4],S[5],S[6],S[7],i);
+ t = S[7]; S[7] = S[6]; S[6] = S[5]; S[5] = S[4];
+ S[4] = S[3]; S[3] = S[2]; S[2] = S[1]; S[1] = S[0]; S[0] = t;
+ }
+
+ /* feedback */
+ for (i = 0; i < 8; i++) {
+ md->sha256.state[i] = md->sha256.state[i] + S[i];
+ }
+}
+
+static void sha256_init(struct hash_state * md)
+{
+ md->sha256.curlen = 0;
+ md->sha256.length = 0;
+ md->sha256.state[0] = 0x6A09E667UL;
+ md->sha256.state[1] = 0xBB67AE85UL;
+ md->sha256.state[2] = 0x3C6EF372UL;
+ md->sha256.state[3] = 0xA54FF53AUL;
+ md->sha256.state[4] = 0x510E527FUL;
+ md->sha256.state[5] = 0x9B05688CUL;
+ md->sha256.state[6] = 0x1F83D9ABUL;
+ md->sha256.state[7] = 0x5BE0CD19UL;
+}
+
+#define MIN(x, y) ( ((x)<(y))?(x):(y) )
+#define SHA256_BLOCKSIZE 64
+static void sha256_process(struct hash_state * md, const unsigned char *in, unsigned long inlen)
+{
+ unsigned long n;
+
+ while (inlen > 0) {
+ if (md->sha256.curlen == 0 && inlen >= SHA256_BLOCKSIZE) {
+ sha256_compress(md, in);
+ md->sha256.length += SHA256_BLOCKSIZE * 8;
+ in += SHA256_BLOCKSIZE;
+ inlen -= SHA256_BLOCKSIZE;
+ } else {
+ n = MIN(inlen, (SHA256_BLOCKSIZE - md->sha256.curlen));
+ memcpy(md->sha256.buf + md->sha256.curlen, in, (size_t)n);
+ md->sha256.curlen += n;
+ in += n;
+ inlen -= n;
+ if (md->sha256.curlen == SHA256_BLOCKSIZE) {
+ sha256_compress(md, md->sha256.buf);
+ md->sha256.length += 8*SHA256_BLOCKSIZE;
+ md->sha256.curlen = 0;
+ }
+ }
+ }
+}
+
+static void sha256_done(struct hash_state * md, unsigned char *out)
+{
+ int i;
+
+ /* increase the length of the message */
+ md->sha256.length += md->sha256.curlen * 8;
+
+ /* append the '1' bit */
+ md->sha256.buf[md->sha256.curlen++] = (unsigned char)0x80;
+
+ /* if the length is currently above 56 bytes we append zeros
+ * then compress. Then we can fall back to padding zeros and length
+ * encoding like normal.
+ */
+ if (md->sha256.curlen > 56) {
+ while (md->sha256.curlen < 64) {
+ md->sha256.buf[md->sha256.curlen++] = (unsigned char)0;
+ }
+ sha256_compress(md, md->sha256.buf);
+ md->sha256.curlen = 0;
+ }
+
+ /* pad upto 56 bytes of zeroes */
+ while (md->sha256.curlen < 56) {
+ md->sha256.buf[md->sha256.curlen++] = (unsigned char)0;
+ }
+
+ /* store length */
+ STORE64H(md->sha256.length, md->sha256.buf+56);
+ sha256_compress(md, md->sha256.buf);
+
+ /* copy output */
+ for (i = 0; i < 8; i++) {
+ STORE32H(md->sha256.state[i], out+(4*i));
+ }
+}
+
+void erofs_sha256(const unsigned char *in, unsigned long in_size,
+ unsigned char out[32])
+{
+ struct hash_state md;
+
+ sha256_init(&md);
+ sha256_process(&md, in, in_size);
+ sha256_done(&md, out);
+}
+
+#ifdef UNITTEST
+static const struct {
+ char *msg;
+ unsigned char hash[32];
+} tests[] = {
+ { "",
+ { 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14,
+ 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24,
+ 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c,
+ 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55 }
+ },
+ { "abc",
+ { 0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea,
+ 0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23,
+ 0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c,
+ 0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad }
+ },
+ { "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq",
+ { 0x24, 0x8d, 0x6a, 0x61, 0xd2, 0x06, 0x38, 0xb8,
+ 0xe5, 0xc0, 0x26, 0x93, 0x0c, 0x3e, 0x60, 0x39,
+ 0xa3, 0x3c, 0xe4, 0x59, 0x64, 0xff, 0x21, 0x67,
+ 0xf6, 0xec, 0xed, 0xd4, 0x19, 0xdb, 0x06, 0xc1 }
+ },
+};
+
+int main(int argc, char **argv)
+{
+ int i;
+ int errors = 0;
+ unsigned char tmp[32];
+
+ for (i = 0; i < (int)(sizeof(tests) / sizeof(tests[0])); i++) {
+ unsigned char *msg = (unsigned char *) tests[i].msg;
+ int len = strlen(tests[i].msg);
+
+ ext2fs_sha256(msg, len, tmp);
+ printf("SHA256 test message %d: ", i);
+ if (memcmp(tmp, tests[i].hash, 32) != 0) {
+ printf("FAILED\n");
+ errors++;
+ } else
+ printf("OK\n");
+ }
+ return errors;
+}
+
+#endif /* UNITTEST */
--
2.24.4
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 3/4] erofs-utils: introduce copy_file_range
2021-08-18 7:03 [PATCH 1/4] erofs-utils: introduce hashmap from git source Gao Xiang
2021-08-18 7:03 ` [PATCH 2/4] erofs-utils: introduce sha256 Gao Xiang
@ 2021-08-18 7:03 ` Gao Xiang
2021-08-18 7:03 ` [PATCH 4/4] erofs-utils: support chunk-based uncompressed files Gao Xiang
2 siblings, 0 replies; 5+ messages in thread
From: Gao Xiang @ 2021-08-18 7:03 UTC (permalink / raw)
To: linux-erofs, Chao Yu, Liu Bo; +Cc: Joseph Qi, Eryu Guan, Liu Jiang, Peng Tao
Add copy_file_range support. Emulate it instead if libc
doesn't support it or have no emulation.
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
configure.ac | 1 +
include/erofs/io.h | 5 +++
lib/io.c | 95 ++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 101 insertions(+)
diff --git a/configure.ac b/configure.ac
index 7217cf531265..af1d06c15996 100644
--- a/configure.ac
+++ b/configure.ac
@@ -158,6 +158,7 @@ AC_CHECK_DECL(lseek64,[AC_DEFINE(HAVE_LSEEK64_PROTOTYPE, 1,
# Checks for library functions.
AC_CHECK_FUNCS(m4_flatten([
backtrace
+ copy_file_range
fallocate
gettimeofday
lgetxattr
diff --git a/include/erofs/io.h b/include/erofs/io.h
index 557424578ece..5b685d217a0f 100644
--- a/include/erofs/io.h
+++ b/include/erofs/io.h
@@ -9,6 +9,7 @@
#ifndef __EROFS_IO_H
#define __EROFS_IO_H
+#define _GNU_SOURCE
#include <unistd.h>
#include "internal.h"
@@ -26,6 +27,10 @@ int dev_fsync(void);
int dev_resize(erofs_blk_t nblocks);
u64 dev_length(void);
+int erofs_copy_file_range(int fd_in, erofs_off_t *off_in,
+ int fd_out, erofs_off_t *off_out,
+ size_t length);
+
static inline int blk_write(const void *buf, erofs_blk_t blkaddr,
u32 nblocks)
{
diff --git a/lib/io.c b/lib/io.c
index 6067041fd829..e4083bb53c27 100644
--- a/lib/io.c
+++ b/lib/io.c
@@ -261,3 +261,98 @@ int dev_read(void *buf, u64 offset, size_t len)
}
return 0;
}
+
+static int __erofs_copy_file_range(int fd_in, erofs_off_t *off_in,
+ int fd_out, erofs_off_t *off_out,
+ size_t length)
+{
+ size_t copied = 0;
+ char buf[8192];
+
+ /*
+ * Main copying loop. The buffer size is arbitrary and is a
+ * trade-off between stack size consumption, cache usage, and
+ * amortization of system call overhead.
+ */
+ while (length > 0) {
+ size_t to_read;
+ ssize_t read_count;
+ char *end, *p;
+
+ to_read = min_t(size_t, length, sizeof(buf));
+#ifdef HAVE_PREAD64
+ read_count = pread64(fd_in, buf, to_read, *off_in);
+#else
+ read_count = pread(fd_in, buf, to_read, *off_in);
+#endif
+ if (read_count == 0)
+ /* End of file reached prematurely. */
+ return copied;
+ if (read_count < 0) {
+ /* Report the number of bytes copied so far. */
+ if (copied > 0)
+ return copied;
+ return -1;
+ }
+ *off_in += read_count;
+
+ /* Write the buffer part which was read to the destination. */
+ end = buf + read_count;
+ for (p = buf; p < end; ) {
+ ssize_t write_count;
+
+#ifdef HAVE_PWRITE64
+ write_count = pwrite64(fd_out, p, end - p, *off_out);
+#else
+ write_count = pwrite(fd_out, p, end - p, *off_out);
+#endif
+ if (write_count < 0) {
+ /*
+ * Adjust the input read position to match what
+ * we have written, so that the caller can pick
+ * up after the error.
+ */
+ size_t written = p - buf;
+ /*
+ * NB: This needs to be signed so that we can
+ * form the negative value below.
+ */
+ ssize_t overread = read_count - written;
+
+ *off_in -= overread;
+ /* Report the number of bytes copied so far. */
+ if (copied + written > 0)
+ return copied + written;
+ return -1;
+ }
+ p += write_count;
+ *off_out += write_count;
+ } /* Write loop. */
+ copied += read_count;
+ length -= read_count;
+ }
+ return copied;
+}
+
+int erofs_copy_file_range(int fd_in, erofs_off_t *off_in,
+ int fd_out, erofs_off_t *off_out,
+ size_t length)
+{
+#ifdef HAVE_COPY_FILE_RANGE
+ off64_t off64_in = *off_in, off64_out = *off_out;
+ ssize_t ret;
+
+ ret = copy_file_range(fd_in, &off64_in, fd_out, &off64_out,
+ length, 0);
+ if (ret >= 0)
+ goto out;
+ if (errno != ENOSYS) {
+ ret = -errno;
+out:
+ *off_in = off64_in;
+ *off_out = off64_out;
+ return ret;
+ }
+#endif
+ return __erofs_copy_file_range(fd_in, off_in, fd_out, off_out, length);
+}
--
2.24.4
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 4/4] erofs-utils: support chunk-based uncompressed files
2021-08-18 7:03 [PATCH 1/4] erofs-utils: introduce hashmap from git source Gao Xiang
2021-08-18 7:03 ` [PATCH 2/4] erofs-utils: introduce sha256 Gao Xiang
2021-08-18 7:03 ` [PATCH 3/4] erofs-utils: introduce copy_file_range Gao Xiang
@ 2021-08-18 7:03 ` Gao Xiang
2021-08-19 6:57 ` [PATCH v2 " Gao Xiang
2 siblings, 1 reply; 5+ messages in thread
From: Gao Xiang @ 2021-08-18 7:03 UTC (permalink / raw)
To: linux-erofs, Chao Yu, Liu Bo; +Cc: Joseph Qi, Eryu Guan, Liu Jiang, Peng Tao
Add mkfs support to new chunk-based uncompressed files.
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
include/erofs/blobchunk.h | 18 ++++
include/erofs/config.h | 2 +-
include/erofs/internal.h | 1 +
include/erofs/io.h | 2 +
include/erofs_fs.h | 39 +++++++-
lib/Makefile.am | 2 +-
lib/blobchunk.c | 200 ++++++++++++++++++++++++++++++++++++++
lib/inode.c | 35 +++++--
lib/io.c | 2 +-
mkfs/main.c | 41 +++++++-
10 files changed, 328 insertions(+), 14 deletions(-)
create mode 100644 include/erofs/blobchunk.h
create mode 100644 lib/blobchunk.c
diff --git a/include/erofs/blobchunk.h b/include/erofs/blobchunk.h
new file mode 100644
index 000000000000..b418227e0ef8
--- /dev/null
+++ b/include/erofs/blobchunk.h
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/blobchunk.h
+ *
+ * Copyright (C) 2021, Alibaba Cloud
+ */
+#ifndef __EROFS_BLOBCHUNK_H
+#define __EROFS_BLOBCHUNK_H
+
+#include "erofs/internal.h"
+
+int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
+int erofs_blob_write_chunked_file(struct erofs_inode *inode);
+int erofs_blob_remap(void);
+void erofs_blob_exit(void);
+int erofs_blob_init(void);
+
+#endif
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 8124f3b36baf..0cd6a359e216 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -44,6 +44,7 @@ struct erofs_configure {
bool c_random_pclusterblks;
#endif
char c_timeinherit;
+ char c_chunkbits;
bool c_noinline_data;
#ifdef HAVE_LIBSELINUX
@@ -57,7 +58,6 @@ struct erofs_configure {
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
int c_inline_xattr_tolerance;
-
u32 c_physical_clusterblks;
u32 c_max_decompressed_extent_bytes;
u64 c_unix_timestamp;
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 5583861b766d..12274769c10d 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -160,6 +160,7 @@ struct erofs_inode {
union {
void *compressmeta;
+ void *chunkindexes;
struct {
uint16_t z_advise;
uint8_t z_algorithmtype[2];
diff --git a/include/erofs/io.h b/include/erofs/io.h
index 5b685d217a0f..226509ff33ca 100644
--- a/include/erofs/io.h
+++ b/include/erofs/io.h
@@ -27,6 +27,8 @@ int dev_fsync(void);
int dev_resize(erofs_blk_t nblocks);
u64 dev_length(void);
+extern int erofs_devfd;
+
int erofs_copy_file_range(int fd_in, erofs_off_t *off_in,
int fd_out, erofs_off_t *off_out,
size_t length);
diff --git a/include/erofs_fs.h b/include/erofs_fs.h
index 18fc1820c58c..b3de05fc5e50 100644
--- a/include/erofs_fs.h
+++ b/include/erofs_fs.h
@@ -5,7 +5,7 @@
*
* Copyright (C) 2017-2018 HUAWEI, Inc.
* http://www.huawei.com/
- * Created by Gao Xiang <gaoxiang25@huawei.com>
+ * Copyright (C) 2021, Alibaba Cloud
*/
#ifndef __EROFS_FS_H
#define __EROFS_FS_H
@@ -22,10 +22,12 @@
#define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x00000001
#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002
#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002
+#define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004
#define EROFS_ALL_FEATURE_INCOMPAT \
(EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \
EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \
- EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER)
+ EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \
+ EROFS_FEATURE_INCOMPAT_CHUNKED_FILE)
#define EROFS_SB_EXTSLOT_SIZE 16
@@ -67,13 +69,16 @@ struct erofs_super_block {
* inode, [xattrs], last_inline_data, ... | ... | no-holed data
* 3 - inode compression D:
* inode, [xattrs], map_header, extents ... | ...
- * 4~7 - reserved
+ * 4 - inode chunk-based E:
+ * inode, [xattrs], chunk indexes ... | ...
+ * 5~7 - reserved
*/
enum {
EROFS_INODE_FLAT_PLAIN = 0,
EROFS_INODE_FLAT_COMPRESSION_LEGACY = 1,
EROFS_INODE_FLAT_INLINE = 2,
EROFS_INODE_FLAT_COMPRESSION = 3,
+ EROFS_INODE_CHUNK_BASED = 4,
EROFS_INODE_DATALAYOUT_MAX
};
@@ -93,6 +98,16 @@ static inline bool erofs_inode_is_data_compressed(unsigned int datamode)
#define EROFS_I_ALL \
((1 << (EROFS_I_DATALAYOUT_BIT + EROFS_I_DATALAYOUT_BITS)) - 1)
+/* indicate chunk blkbits, thus `chunksize = blocksize << chunk blkbits' */
+#define EROFS_CHUNK_FORMAT_BLKBITS 0x001F
+/* with chunk indexes or just a 4-byte blkaddr array */
+#define EROFS_CHUNK_FORMAT_INDEXES 0x0020
+
+struct erofs_inode_chunk_info {
+ __le16 format; /* chunk blkbits */
+ __le16 reserved;
+};
+
/* 32-byte reduced form of an ondisk inode */
struct erofs_inode_compact {
__le16 i_format; /* inode format hints */
@@ -110,6 +125,9 @@ struct erofs_inode_compact {
/* for device files, used to indicate old/new device # */
__le32 rdev;
+
+ /* for chunk-based files, it contains the summary info */
+ struct erofs_inode_chunk_info c;
} i_u;
__le32 i_ino; /* only used for 32-bit stat compatibility */
__le16 i_uid;
@@ -138,6 +156,9 @@ struct erofs_inode_extended {
/* for device files, used to indicate old/new device # */
__le32 rdev;
+
+ /* for chunk-based files, it contains the summary info */
+ struct erofs_inode_chunk_info c;
} i_u;
/* only used for 32-bit stat compatibility */
@@ -207,6 +228,15 @@ static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e)
e->e_name_len + le16_to_cpu(e->e_value_size));
}
+/* represent a zeroed chunk (hole) */
+#define EROFS_NULL_ADDR -1
+
+struct erofs_inode_chunk_index {
+ __le32 blkaddr;
+ __le16 device_id; /* back-end storage id, always 0 for now */
+ __le16 reserved; /* reserved, don't care */
+};
+
/* maximum supported size of a physical compression cluster */
#define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024)
@@ -351,6 +381,8 @@ static inline void erofs_check_ondisk_layout_definitions(void)
BUILD_BUG_ON(sizeof(struct erofs_inode_extended) != 64);
BUILD_BUG_ON(sizeof(struct erofs_xattr_ibody_header) != 12);
BUILD_BUG_ON(sizeof(struct erofs_xattr_entry) != 4);
+ BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_info) != 4);
+ BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_index) != 8);
BUILD_BUG_ON(sizeof(struct z_erofs_map_header) != 8);
BUILD_BUG_ON(sizeof(struct z_erofs_vle_decompressed_index) != 8);
BUILD_BUG_ON(sizeof(struct erofs_dirent) != 12);
@@ -360,4 +392,3 @@ static inline void erofs_check_ondisk_layout_definitions(void)
}
#endif
-
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 8e4ed37f9c70..ea84e0edd628 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -22,7 +22,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
noinst_HEADERS += compressor.h
liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
namei.c data.c compress.c compressor.c zmap.c decompress.c \
- hashmap.c sha256.c
+ hashmap.c sha256.c blobchunk.c
liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
if ENABLE_LZ4
liberofs_la_CFLAGS += ${LZ4_CFLAGS}
diff --git a/lib/blobchunk.c b/lib/blobchunk.c
new file mode 100644
index 000000000000..6ff1c1373e32
--- /dev/null
+++ b/lib/blobchunk.c
@@ -0,0 +1,200 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/blobchunk.c
+ *
+ * Copyright (C) 2021, Alibaba Cloud
+ */
+#define _GNU_SOURCE
+#include "erofs/hashmap.h"
+#include "erofs/blobchunk.h"
+#include "erofs/cache.h"
+#include "erofs/io.h"
+#include <unistd.h>
+
+void erofs_sha256(const unsigned char *in, unsigned long in_size,
+ unsigned char out[32]);
+
+struct erofs_blobchunk {
+ struct hashmap_entry ent;
+ char sha256[32];
+ unsigned int chunksize;
+ erofs_blk_t blkaddr;
+};
+
+static struct hashmap blob_hashmap;
+static FILE *blobfile;
+static erofs_blk_t remapped_base;
+
+static struct erofs_blobchunk *erofs_blob_getchunk(int fd,
+ unsigned int chunksize)
+{
+ static u8 zeroed[EROFS_BLKSIZ];
+ u8 *chunkdata, sha256[32];
+ int ret;
+ unsigned int hash;
+ erofs_off_t blkpos;
+ struct erofs_blobchunk *chunk;
+
+ chunkdata = malloc(chunksize);
+ if (!chunkdata)
+ return ERR_PTR(-ENOMEM);
+
+ ret = read(fd, chunkdata, chunksize);
+ if (ret < chunksize) {
+ chunk = ERR_PTR(-EIO);
+ goto out;
+ }
+ erofs_sha256(chunkdata, chunksize, sha256);
+ hash = memhash(sha256, sizeof(sha256));
+ chunk = hashmap_get_from_hash(&blob_hashmap, hash, sha256);
+ if (chunk) {
+ DBG_BUGON(chunksize != chunk->chunksize);
+ goto out;
+ }
+ chunk = malloc(sizeof(struct erofs_blobchunk));
+ if (!chunk) {
+ chunk = ERR_PTR(-ENOMEM);
+ goto out;
+ }
+
+ chunk->chunksize = chunksize;
+ blkpos = ftell(blobfile);
+ DBG_BUGON(erofs_blkoff(blkpos));
+ chunk->blkaddr = erofs_blknr(blkpos);
+ memcpy(chunk->sha256, sha256, sizeof(sha256));
+ hashmap_entry_init(&chunk->ent, hash);
+ hashmap_add(&blob_hashmap, chunk);
+
+ erofs_dbg("Writing chunk (%u bytes) to %u", chunksize, chunk->blkaddr);
+ ret = fwrite(chunkdata, chunksize, 1, blobfile);
+ if (ret == 1 && erofs_blkoff(chunksize))
+ ret = fwrite(zeroed, EROFS_BLKSIZ - erofs_blkoff(chunksize),
+ 1, blobfile);
+ if (ret < 1) {
+ struct hashmap_entry key;
+
+ hashmap_entry_init(&key, hash);
+ hashmap_remove(&blob_hashmap, &key, sha256);
+ chunk = ERR_PTR(-ENOSPC);
+ goto out;
+ }
+out:
+ free(chunkdata);
+ return chunk;
+}
+
+static int erofs_blob_hashmap_cmp(const void *a, const void *b,
+ const void *key)
+{
+ const struct erofs_blobchunk *ec1 =
+ container_of((struct hashmap_entry *)a,
+ struct erofs_blobchunk, ent);
+ const struct erofs_blobchunk *ec2 =
+ container_of((struct hashmap_entry *)b,
+ struct erofs_blobchunk, ent);
+
+ return memcmp(ec1->sha256, key ? key : ec2->sha256,
+ sizeof(ec1->sha256));
+}
+
+int erofs_blob_write_chunk_indexes(struct erofs_inode *inode,
+ erofs_off_t off)
+{
+ struct erofs_inode_chunk_index idx = {0};
+ unsigned int pos;
+
+ for (pos = 0; pos < inode->extent_isize;
+ pos += sizeof(struct erofs_inode_chunk_index)) {
+ struct erofs_blobchunk *chunk;
+
+ chunk = *(void **)(inode->chunkindexes + pos);
+ idx.blkaddr = chunk->blkaddr + remapped_base;
+ memcpy(inode->chunkindexes + pos, &idx, sizeof(idx));
+ }
+ off = roundup(off, sizeof(struct erofs_inode_chunk_index));
+
+ return dev_write(inode->chunkindexes, off, inode->extent_isize);
+}
+
+int erofs_blob_write_chunked_file(struct erofs_inode *inode)
+{
+ unsigned int chunksize = 1 << cfg.c_chunkbits;
+ unsigned int count = DIV_ROUND_UP(inode->i_size, chunksize);
+ struct erofs_inode_chunk_index *idx;
+ erofs_off_t pos;
+ unsigned int len;
+ int fd, ret;
+
+ inode->extent_isize = count * sizeof(struct erofs_inode_chunk_index);
+ idx = malloc(inode->extent_isize);
+ if (!idx)
+ return -ENOMEM;
+ inode->chunkindexes = idx;
+
+ fd = open(inode->i_srcpath, O_RDONLY | O_BINARY);
+ if (fd < 0) {
+ ret = -errno;
+ goto err;
+ }
+
+ for (pos = 0; pos < inode->i_size; pos += len) {
+ struct erofs_blobchunk *chunk;
+
+ len = min_t(unsigned int, inode->i_size - pos, chunksize);
+ chunk = erofs_blob_getchunk(fd, len);
+ if (IS_ERR(chunk)) {
+ ret = PTR_ERR(chunk);
+ goto err;
+ }
+ *(void **)idx++ = chunk;
+ }
+ inode->datalayout = EROFS_INODE_CHUNK_BASED;
+ close(fd);
+ return 0;
+err:
+ free(inode->chunkindexes);
+ inode->chunkindexes = NULL;
+ return ret;
+}
+
+int erofs_blob_remap(void)
+{
+ struct erofs_buffer_head *bh;
+ ssize_t length;
+ erofs_off_t pos_in, pos_out;
+ int ret;
+
+ fflush(blobfile);
+ length = ftell(blobfile);
+ bh = erofs_balloc(DATA, length, 0, 0);
+ if (IS_ERR(bh))
+ return PTR_ERR(bh);
+
+ erofs_mapbh(bh->block);
+ pos_out = erofs_btell(bh, false);
+ pos_in = 0;
+ remapped_base = erofs_blknr(pos_out);
+ ret = erofs_copy_file_range(fileno(blobfile), &pos_in,
+ erofs_devfd, &pos_out, length);
+ bh->op = &erofs_skip_write_bhops;
+ erofs_bdrop(bh, false);
+ return ret < length ? -EIO : 0;
+}
+
+void erofs_blob_exit(void)
+{
+ if (blobfile)
+ fclose(blobfile);
+
+ hashmap_free(&blob_hashmap, 1);
+}
+
+int erofs_blob_init(void)
+{
+ blobfile = tmpfile64();
+ if (!blobfile)
+ return -ENOMEM;
+
+ hashmap_init(&blob_hashmap, erofs_blob_hashmap_cmp, 0);
+ return 0;
+}
diff --git a/lib/inode.c b/lib/inode.c
index 6871d2be2d49..ac4b833388a8 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -25,6 +25,7 @@
#include "erofs/xattr.h"
#include "erofs/exclude.h"
#include "erofs/block_list.h"
+#include "erofs/blobchunk.h"
#define S_SHIFT 12
static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
@@ -386,6 +387,9 @@ int erofs_write_file(struct erofs_inode *inode)
return 0;
}
+ if (cfg.c_chunkbits)
+ return erofs_blob_write_chunked_file(inode);
+
if (cfg.c_compr_alg_master && erofs_file_is_compressible(inode)) {
ret = erofs_write_compressed_file(inode);
@@ -439,6 +443,11 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)
if (is_inode_layout_compression(inode))
u.dic.i_u.compressed_blocks =
cpu_to_le32(inode->u.i_blocks);
+ else if (inode->datalayout ==
+ EROFS_INODE_CHUNK_BASED)
+ u.dic.i_u.c.format = (cfg.c_chunkbits -
+ LOG_BLOCK_SIZE) |
+ EROFS_CHUNK_FORMAT_INDEXES;
else
u.dic.i_u.raw_blkaddr =
cpu_to_le32(inode->u.i_blkaddr);
@@ -472,6 +481,11 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)
if (is_inode_layout_compression(inode))
u.die.i_u.compressed_blocks =
cpu_to_le32(inode->u.i_blocks);
+ else if (inode->datalayout ==
+ EROFS_INODE_CHUNK_BASED)
+ u.die.i_u.c.format = (cfg.c_chunkbits -
+ LOG_BLOCK_SIZE) |
+ EROFS_CHUNK_FORMAT_INDEXES;
else
u.die.i_u.raw_blkaddr =
cpu_to_le32(inode->u.i_blkaddr);
@@ -504,12 +518,19 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)
}
if (inode->extent_isize) {
- /* write compression metadata */
- off = Z_EROFS_VLE_EXTENT_ALIGN(off);
- ret = dev_write(inode->compressmeta, off, inode->extent_isize);
- if (ret)
- return false;
- free(inode->compressmeta);
+ if (inode->datalayout == EROFS_INODE_CHUNK_BASED) {
+ ret = erofs_blob_write_chunk_indexes(inode, off);
+ if (ret)
+ return false;
+ } else {
+ /* write compression metadata */
+ off = Z_EROFS_VLE_EXTENT_ALIGN(off);
+ ret = dev_write(inode->compressmeta, off,
+ inode->extent_isize);
+ if (ret)
+ return false;
+ free(inode->compressmeta);
+ }
}
inode->bh = NULL;
@@ -564,6 +585,8 @@ static int erofs_prepare_inode_buffer(struct erofs_inode *inode)
if (is_inode_layout_compression(inode))
goto noinline;
+ if (inode->datalayout == EROFS_INODE_CHUNK_BASED)
+ goto noinline;
if (cfg.c_noinline_data && S_ISREG(inode->i_mode)) {
inode->datalayout = EROFS_INODE_FLAT_PLAIN;
diff --git a/lib/io.c b/lib/io.c
index e4083bb53c27..d0d1a3695b6b 100644
--- a/lib/io.c
+++ b/lib/io.c
@@ -26,7 +26,7 @@
#include "erofs/print.h"
static const char *erofs_devname;
-static int erofs_devfd = -1;
+int erofs_devfd = -1;
static u64 erofs_devsz;
int dev_get_blkdev_size(int fd, u64 *bytes)
diff --git a/mkfs/main.c b/mkfs/main.c
index 10fe14d7a722..8c813a229c57 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -23,6 +23,8 @@
#include "erofs/xattr.h"
#include "erofs/exclude.h"
#include "erofs/block_list.h"
+#include "erofs/hashtable.h"
+#include "erofs/blobchunk.h"
#ifdef HAVE_LIBUUID
#include <uuid.h>
@@ -44,6 +46,7 @@ static struct option long_options[] = {
{"random-pclusterblks", no_argument, NULL, 8},
#endif
{"max-extent-bytes", required_argument, NULL, 9},
+ {"chunksize", required_argument, NULL, 256},
#ifdef WITH_ANDROID
{"mount-point", required_argument, NULL, 10},
{"product-out", required_argument, NULL, 11},
@@ -79,6 +82,7 @@ static void usage(void)
#ifdef HAVE_LIBUUID
" -UX use a given filesystem UUID\n"
#endif
+ " --chunksize=X set chunk size to X bytes and use chunk-based files instead\n"
" --exclude-path=X avoid including file X (X = exact literal path)\n"
" --exclude-regex=X avoid including files that match X (X = regular expression)\n"
#ifdef HAVE_LIBSELINUX
@@ -176,7 +180,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
char *endptr;
int opt, i;
- while((opt = getopt_long(argc, argv, "d:x:z:E:T:U:C:",
+ while((opt = getopt_long(argc, argv, "c:d:x:z:E:T:U:C:",
long_options, NULL)) != -1) {
switch (opt) {
case 'z':
@@ -316,6 +320,24 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
}
cfg.c_physical_clusterblks = i / EROFS_BLKSIZ;
break;
+ case 256:
+ i = strtol(optarg, &endptr, 0);
+ if (*endptr != '\0') {
+ erofs_err("invalid chunksize %s", optarg);
+ return -EINVAL;
+ }
+ cfg.c_chunkbits = ilog2(i);
+ if ((1 << cfg.c_chunkbits) != i) {
+ erofs_err("chunksize %s must be a power of two",
+ optarg);
+ return -EINVAL;
+ }
+ if (i < EROFS_BLKSIZ) {
+ erofs_err("chunksize %s must be larger than block size",
+ optarg);
+ return -EINVAL;
+ }
+ break;
case 1:
usage();
@@ -523,6 +545,12 @@ int main(int argc, char **argv)
return 1;
}
+ if (cfg.c_chunkbits) {
+ err = erofs_blob_init();
+ if (err)
+ return 1;
+ }
+
err = lstat64(cfg.c_src_path, &st);
if (err)
return 1;
@@ -610,6 +638,13 @@ int main(int argc, char **argv)
root_nid = erofs_lookupnid(root_inode);
erofs_iput(root_inode);
+ if (cfg.c_chunkbits) {
+ erofs_info("total metadata: %u blocks", erofs_mapbh(NULL));
+ err = erofs_blob_remap();
+ if (err)
+ goto exit;
+ }
+
err = erofs_mkfs_update_super_block(sb_bh, root_nid, &nblocks);
if (err)
goto exit;
@@ -629,6 +664,10 @@ exit:
#endif
dev_close();
erofs_cleanup_exclude_rules();
+
+ if (cfg.c_chunkbits)
+ erofs_blob_exit();
+
erofs_exit_configure();
if (err) {
--
2.24.4
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH v2 4/4] erofs-utils: support chunk-based uncompressed files
2021-08-18 7:03 ` [PATCH 4/4] erofs-utils: support chunk-based uncompressed files Gao Xiang
@ 2021-08-19 6:57 ` Gao Xiang
0 siblings, 0 replies; 5+ messages in thread
From: Gao Xiang @ 2021-08-19 6:57 UTC (permalink / raw)
To: linux-erofs, Chao Yu, Liu Bo; +Cc: Joseph Qi, Eryu Guan, Liu Jiang, Peng Tao
Add mkfs support to new chunk-based uncompressed files.
Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
---
changes since v1:
- update the member order of struct erofs_inode_chunk_index in sync
with the kernel side.
include/erofs/blobchunk.h | 18 ++++
include/erofs/config.h | 2 +-
include/erofs/internal.h | 1 +
include/erofs/io.h | 2 +
include/erofs_fs.h | 42 +++++++-
lib/Makefile.am | 2 +-
lib/blobchunk.c | 200 ++++++++++++++++++++++++++++++++++++++
lib/inode.c | 35 +++++--
lib/io.c | 2 +-
mkfs/main.c | 39 ++++++++
10 files changed, 330 insertions(+), 13 deletions(-)
create mode 100644 include/erofs/blobchunk.h
create mode 100644 lib/blobchunk.c
diff --git a/include/erofs/blobchunk.h b/include/erofs/blobchunk.h
new file mode 100644
index 000000000000..b418227e0ef8
--- /dev/null
+++ b/include/erofs/blobchunk.h
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/blobchunk.h
+ *
+ * Copyright (C) 2021, Alibaba Cloud
+ */
+#ifndef __EROFS_BLOBCHUNK_H
+#define __EROFS_BLOBCHUNK_H
+
+#include "erofs/internal.h"
+
+int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off);
+int erofs_blob_write_chunked_file(struct erofs_inode *inode);
+int erofs_blob_remap(void);
+void erofs_blob_exit(void);
+int erofs_blob_init(void);
+
+#endif
diff --git a/include/erofs/config.h b/include/erofs/config.h
index 8124f3b36baf..0cd6a359e216 100644
--- a/include/erofs/config.h
+++ b/include/erofs/config.h
@@ -44,6 +44,7 @@ struct erofs_configure {
bool c_random_pclusterblks;
#endif
char c_timeinherit;
+ char c_chunkbits;
bool c_noinline_data;
#ifdef HAVE_LIBSELINUX
@@ -57,7 +58,6 @@ struct erofs_configure {
int c_force_inodeversion;
/* < 0, xattr disabled and INT_MAX, always use inline xattrs */
int c_inline_xattr_tolerance;
-
u32 c_physical_clusterblks;
u32 c_max_decompressed_extent_bytes;
u64 c_unix_timestamp;
diff --git a/include/erofs/internal.h b/include/erofs/internal.h
index 5583861b766d..12274769c10d 100644
--- a/include/erofs/internal.h
+++ b/include/erofs/internal.h
@@ -160,6 +160,7 @@ struct erofs_inode {
union {
void *compressmeta;
+ void *chunkindexes;
struct {
uint16_t z_advise;
uint8_t z_algorithmtype[2];
diff --git a/include/erofs/io.h b/include/erofs/io.h
index 5b685d217a0f..226509ff33ca 100644
--- a/include/erofs/io.h
+++ b/include/erofs/io.h
@@ -27,6 +27,8 @@ int dev_fsync(void);
int dev_resize(erofs_blk_t nblocks);
u64 dev_length(void);
+extern int erofs_devfd;
+
int erofs_copy_file_range(int fd_in, erofs_off_t *off_in,
int fd_out, erofs_off_t *off_out,
size_t length);
diff --git a/include/erofs_fs.h b/include/erofs_fs.h
index 18fc1820c58c..4d40010144a7 100644
--- a/include/erofs_fs.h
+++ b/include/erofs_fs.h
@@ -5,7 +5,7 @@
*
* Copyright (C) 2017-2018 HUAWEI, Inc.
* http://www.huawei.com/
- * Created by Gao Xiang <gaoxiang25@huawei.com>
+ * Copyright (C) 2021, Alibaba Cloud
*/
#ifndef __EROFS_FS_H
#define __EROFS_FS_H
@@ -22,10 +22,12 @@
#define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x00000001
#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002
#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002
+#define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004
#define EROFS_ALL_FEATURE_INCOMPAT \
(EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \
EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \
- EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER)
+ EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \
+ EROFS_FEATURE_INCOMPAT_CHUNKED_FILE)
#define EROFS_SB_EXTSLOT_SIZE 16
@@ -67,13 +69,16 @@ struct erofs_super_block {
* inode, [xattrs], last_inline_data, ... | ... | no-holed data
* 3 - inode compression D:
* inode, [xattrs], map_header, extents ... | ...
- * 4~7 - reserved
+ * 4 - inode chunk-based E:
+ * inode, [xattrs], chunk indexes ... | ...
+ * 5~7 - reserved
*/
enum {
EROFS_INODE_FLAT_PLAIN = 0,
EROFS_INODE_FLAT_COMPRESSION_LEGACY = 1,
EROFS_INODE_FLAT_INLINE = 2,
EROFS_INODE_FLAT_COMPRESSION = 3,
+ EROFS_INODE_CHUNK_BASED = 4,
EROFS_INODE_DATALAYOUT_MAX
};
@@ -93,6 +98,16 @@ static inline bool erofs_inode_is_data_compressed(unsigned int datamode)
#define EROFS_I_ALL \
((1 << (EROFS_I_DATALAYOUT_BIT + EROFS_I_DATALAYOUT_BITS)) - 1)
+/* indicate chunk blkbits, thus 'chunksize = blocksize << chunk blkbits' */
+#define EROFS_CHUNK_FORMAT_BLKBITS 0x001F
+/* with chunk indexes or just a 4-byte blkaddr array */
+#define EROFS_CHUNK_FORMAT_INDEXES 0x0020
+
+struct erofs_inode_chunk_info {
+ __le16 format; /* chunk blkbits */
+ __le16 reserved;
+};
+
/* 32-byte reduced form of an ondisk inode */
struct erofs_inode_compact {
__le16 i_format; /* inode format hints */
@@ -110,6 +125,9 @@ struct erofs_inode_compact {
/* for device files, used to indicate old/new device # */
__le32 rdev;
+
+ /* for chunk-based files, it contains the summary info */
+ struct erofs_inode_chunk_info c;
} i_u;
__le32 i_ino; /* only used for 32-bit stat compatibility */
__le16 i_uid;
@@ -138,6 +156,9 @@ struct erofs_inode_extended {
/* for device files, used to indicate old/new device # */
__le32 rdev;
+
+ /* for chunk-based files, it contains the summary info */
+ struct erofs_inode_chunk_info c;
} i_u;
/* only used for 32-bit stat compatibility */
@@ -207,6 +228,15 @@ static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e)
e->e_name_len + le16_to_cpu(e->e_value_size));
}
+/* represent a zeroed chunk (hole) */
+#define EROFS_NULL_ADDR -1
+
+struct erofs_inode_chunk_index {
+ __le16 advise; /* always 0, don't care for now */
+ __le16 device_id; /* back-end storage id, always 0 for now */
+ __le32 blkaddr; /* start block address of this inode chunk */
+};
+
/* maximum supported size of a physical compression cluster */
#define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024)
@@ -351,13 +381,17 @@ static inline void erofs_check_ondisk_layout_definitions(void)
BUILD_BUG_ON(sizeof(struct erofs_inode_extended) != 64);
BUILD_BUG_ON(sizeof(struct erofs_xattr_ibody_header) != 12);
BUILD_BUG_ON(sizeof(struct erofs_xattr_entry) != 4);
+ BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_info) != 4);
+ BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_index) != 8);
BUILD_BUG_ON(sizeof(struct z_erofs_map_header) != 8);
BUILD_BUG_ON(sizeof(struct z_erofs_vle_decompressed_index) != 8);
BUILD_BUG_ON(sizeof(struct erofs_dirent) != 12);
+ /* keep in sync between 2 index structures for better extendibility */
+ BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_index) !=
+ sizeof(struct z_erofs_vle_decompressed_index));
BUILD_BUG_ON(BIT(Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) <
Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1);
}
#endif
-
diff --git a/lib/Makefile.am b/lib/Makefile.am
index 8e4ed37f9c70..ea84e0edd628 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -22,7 +22,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
noinst_HEADERS += compressor.h
liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
namei.c data.c compress.c compressor.c zmap.c decompress.c \
- hashmap.c sha256.c
+ hashmap.c sha256.c blobchunk.c
liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include
if ENABLE_LZ4
liberofs_la_CFLAGS += ${LZ4_CFLAGS}
diff --git a/lib/blobchunk.c b/lib/blobchunk.c
new file mode 100644
index 000000000000..6ff1c1373e32
--- /dev/null
+++ b/lib/blobchunk.c
@@ -0,0 +1,200 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * erofs-utils/lib/blobchunk.c
+ *
+ * Copyright (C) 2021, Alibaba Cloud
+ */
+#define _GNU_SOURCE
+#include "erofs/hashmap.h"
+#include "erofs/blobchunk.h"
+#include "erofs/cache.h"
+#include "erofs/io.h"
+#include <unistd.h>
+
+void erofs_sha256(const unsigned char *in, unsigned long in_size,
+ unsigned char out[32]);
+
+struct erofs_blobchunk {
+ struct hashmap_entry ent;
+ char sha256[32];
+ unsigned int chunksize;
+ erofs_blk_t blkaddr;
+};
+
+static struct hashmap blob_hashmap;
+static FILE *blobfile;
+static erofs_blk_t remapped_base;
+
+static struct erofs_blobchunk *erofs_blob_getchunk(int fd,
+ unsigned int chunksize)
+{
+ static u8 zeroed[EROFS_BLKSIZ];
+ u8 *chunkdata, sha256[32];
+ int ret;
+ unsigned int hash;
+ erofs_off_t blkpos;
+ struct erofs_blobchunk *chunk;
+
+ chunkdata = malloc(chunksize);
+ if (!chunkdata)
+ return ERR_PTR(-ENOMEM);
+
+ ret = read(fd, chunkdata, chunksize);
+ if (ret < chunksize) {
+ chunk = ERR_PTR(-EIO);
+ goto out;
+ }
+ erofs_sha256(chunkdata, chunksize, sha256);
+ hash = memhash(sha256, sizeof(sha256));
+ chunk = hashmap_get_from_hash(&blob_hashmap, hash, sha256);
+ if (chunk) {
+ DBG_BUGON(chunksize != chunk->chunksize);
+ goto out;
+ }
+ chunk = malloc(sizeof(struct erofs_blobchunk));
+ if (!chunk) {
+ chunk = ERR_PTR(-ENOMEM);
+ goto out;
+ }
+
+ chunk->chunksize = chunksize;
+ blkpos = ftell(blobfile);
+ DBG_BUGON(erofs_blkoff(blkpos));
+ chunk->blkaddr = erofs_blknr(blkpos);
+ memcpy(chunk->sha256, sha256, sizeof(sha256));
+ hashmap_entry_init(&chunk->ent, hash);
+ hashmap_add(&blob_hashmap, chunk);
+
+ erofs_dbg("Writing chunk (%u bytes) to %u", chunksize, chunk->blkaddr);
+ ret = fwrite(chunkdata, chunksize, 1, blobfile);
+ if (ret == 1 && erofs_blkoff(chunksize))
+ ret = fwrite(zeroed, EROFS_BLKSIZ - erofs_blkoff(chunksize),
+ 1, blobfile);
+ if (ret < 1) {
+ struct hashmap_entry key;
+
+ hashmap_entry_init(&key, hash);
+ hashmap_remove(&blob_hashmap, &key, sha256);
+ chunk = ERR_PTR(-ENOSPC);
+ goto out;
+ }
+out:
+ free(chunkdata);
+ return chunk;
+}
+
+static int erofs_blob_hashmap_cmp(const void *a, const void *b,
+ const void *key)
+{
+ const struct erofs_blobchunk *ec1 =
+ container_of((struct hashmap_entry *)a,
+ struct erofs_blobchunk, ent);
+ const struct erofs_blobchunk *ec2 =
+ container_of((struct hashmap_entry *)b,
+ struct erofs_blobchunk, ent);
+
+ return memcmp(ec1->sha256, key ? key : ec2->sha256,
+ sizeof(ec1->sha256));
+}
+
+int erofs_blob_write_chunk_indexes(struct erofs_inode *inode,
+ erofs_off_t off)
+{
+ struct erofs_inode_chunk_index idx = {0};
+ unsigned int pos;
+
+ for (pos = 0; pos < inode->extent_isize;
+ pos += sizeof(struct erofs_inode_chunk_index)) {
+ struct erofs_blobchunk *chunk;
+
+ chunk = *(void **)(inode->chunkindexes + pos);
+ idx.blkaddr = chunk->blkaddr + remapped_base;
+ memcpy(inode->chunkindexes + pos, &idx, sizeof(idx));
+ }
+ off = roundup(off, sizeof(struct erofs_inode_chunk_index));
+
+ return dev_write(inode->chunkindexes, off, inode->extent_isize);
+}
+
+int erofs_blob_write_chunked_file(struct erofs_inode *inode)
+{
+ unsigned int chunksize = 1 << cfg.c_chunkbits;
+ unsigned int count = DIV_ROUND_UP(inode->i_size, chunksize);
+ struct erofs_inode_chunk_index *idx;
+ erofs_off_t pos;
+ unsigned int len;
+ int fd, ret;
+
+ inode->extent_isize = count * sizeof(struct erofs_inode_chunk_index);
+ idx = malloc(inode->extent_isize);
+ if (!idx)
+ return -ENOMEM;
+ inode->chunkindexes = idx;
+
+ fd = open(inode->i_srcpath, O_RDONLY | O_BINARY);
+ if (fd < 0) {
+ ret = -errno;
+ goto err;
+ }
+
+ for (pos = 0; pos < inode->i_size; pos += len) {
+ struct erofs_blobchunk *chunk;
+
+ len = min_t(unsigned int, inode->i_size - pos, chunksize);
+ chunk = erofs_blob_getchunk(fd, len);
+ if (IS_ERR(chunk)) {
+ ret = PTR_ERR(chunk);
+ goto err;
+ }
+ *(void **)idx++ = chunk;
+ }
+ inode->datalayout = EROFS_INODE_CHUNK_BASED;
+ close(fd);
+ return 0;
+err:
+ free(inode->chunkindexes);
+ inode->chunkindexes = NULL;
+ return ret;
+}
+
+int erofs_blob_remap(void)
+{
+ struct erofs_buffer_head *bh;
+ ssize_t length;
+ erofs_off_t pos_in, pos_out;
+ int ret;
+
+ fflush(blobfile);
+ length = ftell(blobfile);
+ bh = erofs_balloc(DATA, length, 0, 0);
+ if (IS_ERR(bh))
+ return PTR_ERR(bh);
+
+ erofs_mapbh(bh->block);
+ pos_out = erofs_btell(bh, false);
+ pos_in = 0;
+ remapped_base = erofs_blknr(pos_out);
+ ret = erofs_copy_file_range(fileno(blobfile), &pos_in,
+ erofs_devfd, &pos_out, length);
+ bh->op = &erofs_skip_write_bhops;
+ erofs_bdrop(bh, false);
+ return ret < length ? -EIO : 0;
+}
+
+void erofs_blob_exit(void)
+{
+ if (blobfile)
+ fclose(blobfile);
+
+ hashmap_free(&blob_hashmap, 1);
+}
+
+int erofs_blob_init(void)
+{
+ blobfile = tmpfile64();
+ if (!blobfile)
+ return -ENOMEM;
+
+ hashmap_init(&blob_hashmap, erofs_blob_hashmap_cmp, 0);
+ return 0;
+}
diff --git a/lib/inode.c b/lib/inode.c
index 6871d2be2d49..ac4b833388a8 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -25,6 +25,7 @@
#include "erofs/xattr.h"
#include "erofs/exclude.h"
#include "erofs/block_list.h"
+#include "erofs/blobchunk.h"
#define S_SHIFT 12
static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = {
@@ -386,6 +387,9 @@ int erofs_write_file(struct erofs_inode *inode)
return 0;
}
+ if (cfg.c_chunkbits)
+ return erofs_blob_write_chunked_file(inode);
+
if (cfg.c_compr_alg_master && erofs_file_is_compressible(inode)) {
ret = erofs_write_compressed_file(inode);
@@ -439,6 +443,11 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)
if (is_inode_layout_compression(inode))
u.dic.i_u.compressed_blocks =
cpu_to_le32(inode->u.i_blocks);
+ else if (inode->datalayout ==
+ EROFS_INODE_CHUNK_BASED)
+ u.dic.i_u.c.format = (cfg.c_chunkbits -
+ LOG_BLOCK_SIZE) |
+ EROFS_CHUNK_FORMAT_INDEXES;
else
u.dic.i_u.raw_blkaddr =
cpu_to_le32(inode->u.i_blkaddr);
@@ -472,6 +481,11 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)
if (is_inode_layout_compression(inode))
u.die.i_u.compressed_blocks =
cpu_to_le32(inode->u.i_blocks);
+ else if (inode->datalayout ==
+ EROFS_INODE_CHUNK_BASED)
+ u.die.i_u.c.format = (cfg.c_chunkbits -
+ LOG_BLOCK_SIZE) |
+ EROFS_CHUNK_FORMAT_INDEXES;
else
u.die.i_u.raw_blkaddr =
cpu_to_le32(inode->u.i_blkaddr);
@@ -504,12 +518,19 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh)
}
if (inode->extent_isize) {
- /* write compression metadata */
- off = Z_EROFS_VLE_EXTENT_ALIGN(off);
- ret = dev_write(inode->compressmeta, off, inode->extent_isize);
- if (ret)
- return false;
- free(inode->compressmeta);
+ if (inode->datalayout == EROFS_INODE_CHUNK_BASED) {
+ ret = erofs_blob_write_chunk_indexes(inode, off);
+ if (ret)
+ return false;
+ } else {
+ /* write compression metadata */
+ off = Z_EROFS_VLE_EXTENT_ALIGN(off);
+ ret = dev_write(inode->compressmeta, off,
+ inode->extent_isize);
+ if (ret)
+ return false;
+ free(inode->compressmeta);
+ }
}
inode->bh = NULL;
@@ -564,6 +585,8 @@ static int erofs_prepare_inode_buffer(struct erofs_inode *inode)
if (is_inode_layout_compression(inode))
goto noinline;
+ if (inode->datalayout == EROFS_INODE_CHUNK_BASED)
+ goto noinline;
if (cfg.c_noinline_data && S_ISREG(inode->i_mode)) {
inode->datalayout = EROFS_INODE_FLAT_PLAIN;
diff --git a/lib/io.c b/lib/io.c
index e4083bb53c27..d0d1a3695b6b 100644
--- a/lib/io.c
+++ b/lib/io.c
@@ -26,7 +26,7 @@
#include "erofs/print.h"
static const char *erofs_devname;
-static int erofs_devfd = -1;
+int erofs_devfd = -1;
static u64 erofs_devsz;
int dev_get_blkdev_size(int fd, u64 *bytes)
diff --git a/mkfs/main.c b/mkfs/main.c
index 10fe14d7a722..aa634e9263b4 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -23,6 +23,8 @@
#include "erofs/xattr.h"
#include "erofs/exclude.h"
#include "erofs/block_list.h"
+#include "erofs/hashtable.h"
+#include "erofs/blobchunk.h"
#ifdef HAVE_LIBUUID
#include <uuid.h>
@@ -44,6 +46,7 @@ static struct option long_options[] = {
{"random-pclusterblks", no_argument, NULL, 8},
#endif
{"max-extent-bytes", required_argument, NULL, 9},
+ {"chunksize", required_argument, NULL, 256},
#ifdef WITH_ANDROID
{"mount-point", required_argument, NULL, 10},
{"product-out", required_argument, NULL, 11},
@@ -79,6 +82,7 @@ static void usage(void)
#ifdef HAVE_LIBUUID
" -UX use a given filesystem UUID\n"
#endif
+ " --chunksize=X set chunk size to X bytes and use chunk-based files instead\n"
" --exclude-path=X avoid including file X (X = exact literal path)\n"
" --exclude-regex=X avoid including files that match X (X = regular expression)\n"
#ifdef HAVE_LIBSELINUX
@@ -316,6 +320,24 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
}
cfg.c_physical_clusterblks = i / EROFS_BLKSIZ;
break;
+ case 256:
+ i = strtol(optarg, &endptr, 0);
+ if (*endptr != '\0') {
+ erofs_err("invalid chunksize %s", optarg);
+ return -EINVAL;
+ }
+ cfg.c_chunkbits = ilog2(i);
+ if ((1 << cfg.c_chunkbits) != i) {
+ erofs_err("chunksize %s must be a power of two",
+ optarg);
+ return -EINVAL;
+ }
+ if (i < EROFS_BLKSIZ) {
+ erofs_err("chunksize %s must be larger than block size",
+ optarg);
+ return -EINVAL;
+ }
+ break;
case 1:
usage();
@@ -523,6 +545,12 @@ int main(int argc, char **argv)
return 1;
}
+ if (cfg.c_chunkbits) {
+ err = erofs_blob_init();
+ if (err)
+ return 1;
+ }
+
err = lstat64(cfg.c_src_path, &st);
if (err)
return 1;
@@ -610,6 +638,13 @@ int main(int argc, char **argv)
root_nid = erofs_lookupnid(root_inode);
erofs_iput(root_inode);
+ if (cfg.c_chunkbits) {
+ erofs_info("total metadata: %u blocks", erofs_mapbh(NULL));
+ err = erofs_blob_remap();
+ if (err)
+ goto exit;
+ }
+
err = erofs_mkfs_update_super_block(sb_bh, root_nid, &nblocks);
if (err)
goto exit;
@@ -629,6 +664,10 @@ exit:
#endif
dev_close();
erofs_cleanup_exclude_rules();
+
+ if (cfg.c_chunkbits)
+ erofs_blob_exit();
+
erofs_exit_configure();
if (err) {
--
2.24.4
^ permalink raw reply related [flat|nested] 5+ messages in thread