[PATCH 0/5] mtd-utils: Add fastmap support to ubinize

[PATCH 0/5] mtd-utils: Add fastmap support to ubinize

[Richard Weinberger]

This is the userspace side of my preseeded fastmap series.
ubinize can now created images with a preloaded fastmap on it.

Richard Weinberger (5):
  Sync ubi-media.h with Linux
  ubinize: Don't stat image file twice
  libubigen: Carry image path and size in volume info
  libubigen: Maintain a eraseblock association table
  ubinize: Add fastmap support

 include/libubigen.h     |  25 ++++-
 include/mtd/ubi-media.h | 158 +++++++++++++++++++++++++++++--
 lib/libubigen.c         | 241 ++++++++++++++++++++++++++++++++++++++++++++----
 ubi-utils/ubiformat.c   |   6 +-
 ubi-utils/ubinize.c     | 120 ++++++++++++++++--------
 5 files changed, 477 insertions(+), 73 deletions(-)

-- 
2.13.6

[PATCH 1/5] Sync ubi-media.h with Linux

[Richard Weinberger]

...to get UBI Fastmap on-fast layout.

Signed-off-by: Richard Weinberger <richard@nod.at>
---
 include/mtd/ubi-media.h | 158 ++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 152 insertions(+), 6 deletions(-)

diff --git a/include/mtd/ubi-media.h b/include/mtd/ubi-media.h
index 08bec3ed96f0..a2c471210202 100644
--- a/include/mtd/ubi-media.h
+++ b/include/mtd/ubi-media.h
@@ -149,10 +149,10 @@ enum {
  * The @image_seq field is used to validate a UBI image that has been prepared
  * for a UBI device. The @image_seq value can be any value, but it must be the
  * same on all eraseblocks. UBI will ensure that all new erase counter headers
- * also contain this value, and will check the value when scanning at start-up.
+ * also contain this value, and will check the value when attaching the flash.
  * One way to make use of @image_seq is to increase its value by one every time
  * an image is flashed over an existing image, then, if the flashing does not
- * complete, UBI will detect the error when scanning.
+ * complete, UBI will detect the error when attaching the media.
  */
 struct ubi_ec_hdr {
 	__be32  magic;
@@ -229,7 +229,7 @@ struct ubi_ec_hdr {
  * copy. UBI also calculates data CRC when the data is moved and stores it at
  * the @data_crc field of the copy (P1). So when UBI needs to pick one physical
  * eraseblock of two (P or P1), the @copy_flag of the newer one (P1) is
- * examined. If it is cleared, the situation* is simple and the newer one is
+ * examined. If it is cleared, the situation is simple and the newer one is
  * picked. If it is set, the data CRC of the copy (P1) is examined. If the CRC
  * checksum is correct, this physical eraseblock is selected (P1). Otherwise
  * the older one (P) is selected.
@@ -283,7 +283,7 @@ struct ubi_vid_hdr {
 	__u8    compat;
 	__be32  vol_id;
 	__be32  lnum;
-	__be32  leb_ver;
+	__u8    padding1[4];
 	__be32  data_size;
 	__be32  used_ebs;
 	__be32  data_pad;
@@ -298,8 +298,8 @@ struct ubi_vid_hdr {
 #define UBI_INT_VOL_COUNT 1
 
 /*
- * Starting ID of internal volumes. There is reserved room for 4096 internal
- * volumes.
+ * Starting ID of internal volumes: 0x7fffefff.
+ * There is reserved room for 4096 internal volumes.
  */
 #define UBI_INTERNAL_VOL_START (0x7FFFFFFF - 4096)
 
@@ -375,4 +375,150 @@ struct ubi_vtbl_record {
 	__be32  crc;
 } __attribute__ ((packed));
 
+/* UBI fastmap on-flash data structures */
+
+#define UBI_FM_SB_VOLUME_ID	(UBI_LAYOUT_VOLUME_ID + 1)
+#define UBI_FM_DATA_VOLUME_ID	(UBI_LAYOUT_VOLUME_ID + 2)
+
+/* fastmap on-flash data structure format version */
+#define UBI_FM_FMT_VERSION		2
+
+/* this implementation writes always version 1 */
+#define UBI_FM_FMT_WRITE_VERSION	1
+
+#define UBI_FM_SB_MAGIC		0x7B11D69F
+#define UBI_FM_HDR_MAGIC	0xD4B82EF7
+#define UBI_FM_VHDR_MAGIC	0xFA370ED1
+#define UBI_FM_POOL_MAGIC	0x67AF4D08
+#define UBI_FM_EBA_MAGIC	0xf0c040a8
+
+/* A fastmap super block can be located between PEB 0 and
+ * UBI_FM_MAX_START */
+#define UBI_FM_MAX_START	64
+
+/* A fastmap can use up to UBI_FM_MAX_BLOCKS PEBs */
+#define UBI_FM_MAX_BLOCKS	32
+
+/* 5% of the total number of PEBs have to be scanned while attaching
+ * from a fastmap.
+ * But the size of this pool is limited to be between UBI_FM_MIN_POOL_SIZE and
+ * UBI_FM_MAX_POOL_SIZE */
+#define UBI_FM_MIN_POOL_SIZE	8
+#define UBI_FM_MAX_POOL_SIZE	256
+
+enum {
+	UBI_FM_SB_PRESEEDED_FLG	= 0x1,
+};
+
+#define UBI_FM_SB_FLG_MASK (UBI_FM_SB_PRESEEDED_FLG)
+
+/**
+ * struct ubi_fm_sb - UBI fastmap super block
+ * @magic: fastmap super block magic number (%UBI_FM_SB_MAGIC)
+ * @version: format version of this fastmap
+ * @data_crc: CRC over the fastmap data
+ * @used_blocks: number of PEBs used by this fastmap
+ * @block_loc: an array containing the location of all PEBs of the fastmap
+ * @block_ec: the erase counter of each used PEB
+ * @sqnum: highest sequence number value at the time while taking the fastmap
+ * @flags: fastmap specific flags, only used with @version > 1, zero otherwise
+ *
+ */
+struct ubi_fm_sb {
+	__be32 magic;
+	__u8 version;
+	__u8 padding1[3];
+	__be32 data_crc;
+	__be32 used_blocks;
+	__be32 block_loc[UBI_FM_MAX_BLOCKS];
+	__be32 block_ec[UBI_FM_MAX_BLOCKS];
+	__be64 sqnum;
+	__be32 flags;
+	__u8 padding2[28];
+} __attribute__ ((packed));
+
+/**
+ * struct ubi_fm_hdr - header of the fastmap data set
+ * @magic: fastmap header magic number (%UBI_FM_HDR_MAGIC)
+ * @free_peb_count: number of free PEBs known by this fastmap
+ * @used_peb_count: number of used PEBs known by this fastmap
+ * @scrub_peb_count: number of to be scrubbed PEBs known by this fastmap
+ * @bad_peb_count: number of bad PEBs known by this fastmap
+ * @erase_peb_count: number of bad PEBs which have to be erased
+ * @vol_count: number of UBI volumes known by this fastmap
+ */
+struct ubi_fm_hdr {
+	__be32 magic;
+	__be32 free_peb_count;
+	__be32 used_peb_count;
+	__be32 scrub_peb_count;
+	__be32 bad_peb_count;
+	__be32 erase_peb_count;
+	__be32 vol_count;
+	__u8 padding[4];
+} __attribute__ ((packed));
+
+/* struct ubi_fm_hdr is followed by two struct ubi_fm_scan_pool */
+
+/**
+ * struct ubi_fm_scan_pool - Fastmap pool PEBs to be scanned while attaching
+ * @magic: pool magic numer (%UBI_FM_POOL_MAGIC)
+ * @size: current pool size
+ * @max_size: maximal pool size
+ * @pebs: an array containing the location of all PEBs in this pool
+ */
+struct ubi_fm_scan_pool {
+	__be32 magic;
+	__be16 size;
+	__be16 max_size;
+	__be32 pebs[UBI_FM_MAX_POOL_SIZE];
+	__be32 padding[4];
+} __attribute__ ((packed));
+
+/* ubi_fm_scan_pool is followed by nfree+nused struct ubi_fm_ec records */
+
+/**
+ * struct ubi_fm_ec - stores the erase counter of a PEB
+ * @pnum: PEB number
+ * @ec: ec of this PEB
+ */
+struct ubi_fm_ec {
+	__be32 pnum;
+	__be32 ec;
+} __attribute__ ((packed));
+
+/**
+ * struct ubi_fm_volhdr - Fastmap volume header
+ * it identifies the start of an eba table
+ * @magic: Fastmap volume header magic number (%UBI_FM_VHDR_MAGIC)
+ * @vol_id: volume id of the fastmapped volume
+ * @vol_type: type of the fastmapped volume
+ * @data_pad: data_pad value of the fastmapped volume
+ * @used_ebs: number of used LEBs within this volume
+ * @last_eb_bytes: number of bytes used in the last LEB
+ */
+struct ubi_fm_volhdr {
+	__be32 magic;
+	__be32 vol_id;
+	__u8 vol_type;
+	__u8 padding1[3];
+	__be32 data_pad;
+	__be32 used_ebs;
+	__be32 last_eb_bytes;
+	__u8 padding2[8];
+} __attribute__ ((packed));
+
+/* struct ubi_fm_volhdr is followed by one struct ubi_fm_eba records */
+
+/**
+ * struct ubi_fm_eba - denotes an association between a PEB and LEB
+ * @magic: EBA table magic number
+ * @reserved_pebs: number of table entries
+ * @pnum: PEB number of LEB (LEB is the index)
+ */
+struct ubi_fm_eba {
+	__be32 magic;
+	__be32 reserved_pebs;
+	__be32 pnum[0];
+} __attribute__ ((packed));
 #endif /* !__UBI_MEDIA_H__ */
-- 
2.13.6

[PATCH 2/5] ubinize: Don't stat image file twice

[Richard Weinberger]

Signed-off-by: Richard Weinberger <richard@nod.at>
---
 ubi-utils/ubinize.c | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/ubi-utils/ubinize.c b/ubi-utils/ubinize.c
index c85ff9b22c58..3390d0d95305 100644
--- a/ubi-utils/ubinize.c
+++ b/ubi-utils/ubinize.c
@@ -335,16 +335,11 @@ static int read_section(const struct ubigen_info *ui, const char *sname,
 				      sname, *img, (long long)st->st_size, vi->bytes);
 		verbose(args.verbose, "volume size: %lld bytes", vi->bytes);
 	} else {
-		struct stat st;
-
 		if (!*img)
 			return errmsg("neither image file (\"image=\") nor volume size "
 				      "(\"vol_size=\") specified in section \"%s\"", sname);
 
-		if (stat(*img, &st))
-			return sys_errmsg("cannot stat \"%s\"", *img);
-
-		vi->bytes = st.st_size;
+		vi->bytes = st->st_size;
 
 		if (vi->bytes == 0)
 			return errmsg("file \"%s\" referred from section \"%s\" is empty",
-- 
2.13.6

[PATCH 3/5] libubigen: Carry image path and size in volume info

[Richard Weinberger]

...instead of passing them a extra parameters.

Signed-off-by: Richard Weinberger <richard@nod.at>
---
 include/libubigen.h |  7 +++++--
 lib/libubigen.c     |  3 ++-
 ubi-utils/ubinize.c | 41 ++++++++++++++++++++---------------------
 3 files changed, 27 insertions(+), 24 deletions(-)

diff --git a/include/libubigen.h b/include/libubigen.h
index c25ac20fbdc2..8084d0ecddb2 100644
--- a/include/libubigen.h
+++ b/include/libubigen.h
@@ -74,6 +74,8 @@ struct ubigen_info
  * @bytes: size of the volume contents in bytes (relevant for static volumes
  *         only)
  * @flags: volume flags (%UBI_VTBL_AUTORESIZE_FLG)
+ * @image_file: file to read volume contents from, can be NULL
+ * @image_file_len: length of %image_file in bytes
  */
 struct ubigen_vol_info
 {
@@ -88,6 +90,8 @@ struct ubigen_vol_info
 	int used_ebs;
 	long long bytes;
 	uint8_t flags;
+	const char *image_file;
+	size_t image_file_len;
 };
 
 /**
@@ -159,7 +163,6 @@ int ubigen_add_volume(const struct ubigen_info *ui,
  * @ui: libubigen information
  * @vi: volume information
  * @ec: erase counter value to put to EC headers
- * @bytes: volume size in bytes
  * @in: input file descriptor (has to be properly seeked)
  * @out: output file descriptor
  *
@@ -169,7 +172,7 @@ int ubigen_add_volume(const struct ubigen_info *ui,
  */
 int ubigen_write_volume(const struct ubigen_info *ui,
 			const struct ubigen_vol_info *vi, long long ec,
-			long long bytes, int in, int out);
+			int in, int out);
 
 /**
  * ubigen_write_layout_vol - write UBI layout volume
diff --git a/lib/libubigen.c b/lib/libubigen.c
index d2a949be90c2..900c984229fb 100644
--- a/lib/libubigen.c
+++ b/lib/libubigen.c
@@ -170,9 +170,10 @@ void ubigen_init_vid_hdr(const struct ubigen_info *ui,
 
 int ubigen_write_volume(const struct ubigen_info *ui,
 			const struct ubigen_vol_info *vi, long long ec,
-			long long bytes, int in, int out)
+			int in, int out)
 {
 	int len = vi->usable_leb_size, rd, lnum = 0;
+	long long bytes = vi->image_file_len;
 	char *inbuf, *outbuf;
 
 	if (vi->id >= ui->max_volumes) {
diff --git a/ubi-utils/ubinize.c b/ubi-utils/ubinize.c
index 3390d0d95305..bffb66ebcbd9 100644
--- a/ubi-utils/ubinize.c
+++ b/ubi-utils/ubinize.c
@@ -241,13 +241,13 @@ static int parse_opt(int argc, char * const argv[])
 }
 
 static int read_section(const struct ubigen_info *ui, const char *sname,
-			struct ubigen_vol_info *vi, const char **img,
-			struct stat *st)
+			struct ubigen_vol_info *vi)
 {
 	char buf[256];
 	const char *p;
+	struct stat st;
 
-	*img = NULL;
+	vi->image_file = NULL;
 
 	if (strlen(sname) > 128)
 		return errmsg("too long section name \"%s\"", sname);
@@ -294,13 +294,14 @@ static int read_section(const struct ubigen_info *ui, const char *sname,
 	sprintf(buf, "%s:image", sname);
 	p = iniparser_getstring(args.dict, buf, NULL);
 	if (p) {
-		*img = p;
-		if (stat(p, st))
+		vi->image_file = p;
+		if (stat(p, &st))
 			return sys_errmsg("cannot stat \"%s\" referred from section \"%s\"",
 					  p, sname);
-		if (st->st_size == 0)
+		if (st.st_size == 0)
 			return errmsg("empty file \"%s\" referred from section \"%s\"",
 				       p, sname);
+		vi->image_file_len = st.st_size;
 	} else if (vi->type == UBI_VID_STATIC)
 		return errmsg("image is not specified for static volume in section \"%s\"",
 			      sname);
@@ -329,24 +330,24 @@ static int read_section(const struct ubigen_info *ui, const char *sname,
 				      p, sname);
 
 		/* Make sure the image size is not larger than volume size */
-		if (*img && st->st_size > vi->bytes)
+		if (vi->image_file && st.st_size > vi->bytes)
 			return errmsg("error in section \"%s\": size of the image file "
 				      "\"%s\" is %lld, which is larger than volume size %lld",
-				      sname, *img, (long long)st->st_size, vi->bytes);
+				      sname, vi->image_file, (long long)st.st_size, vi->bytes);
 		verbose(args.verbose, "volume size: %lld bytes", vi->bytes);
 	} else {
-		if (!*img)
+		if (!vi->image_file)
 			return errmsg("neither image file (\"image=\") nor volume size "
 				      "(\"vol_size=\") specified in section \"%s\"", sname);
 
-		vi->bytes = st->st_size;
+		vi->bytes = st.st_size;
 
 		if (vi->bytes == 0)
 			return errmsg("file \"%s\" referred from section \"%s\" is empty",
-				      *img, sname);
+				      vi->image_file, sname);
 
 		normsg_cont("volume size was not specified in section \"%s\", assume"
-			    " minimum to fit image \"%s\"", sname, *img);
+			    " minimum to fit image \"%s\"", sname, vi->image_file);
 		util_print_bytes(vi->bytes, 1);
 		printf("\n");
 	}
@@ -395,7 +396,7 @@ static int read_section(const struct ubigen_info *ui, const char *sname,
 	if (vi->type == UBI_VID_DYNAMIC)
 		vi->used_ebs = (vi->bytes + vi->usable_leb_size - 1) / vi->usable_leb_size;
 	else
-		vi->used_ebs = (st->st_size + vi->usable_leb_size - 1) / vi->usable_leb_size;
+		vi->used_ebs = (st.st_size + vi->usable_leb_size - 1) / vi->usable_leb_size;
 	vi->compat = 0;
 	return 0;
 }
@@ -476,8 +477,6 @@ int main(int argc, char * const argv[])
 
 	for (i = 0; i < sects; i++) {
 		const char *sname = iniparser_getsecname(args.dict, i);
-		const char *img = NULL;
-		struct stat st;
 		int fd, j;
 
 		if (!sname) {
@@ -490,7 +489,7 @@ int main(int argc, char * const argv[])
 			printf("\n");
 		verbose(args.verbose, "parsing section \"%s\"", sname);
 
-		err = read_section(&ui, sname, &vi[i], &img, &st);
+		err = read_section(&ui, sname, &vi[i]);
 		if (err == -1)
 			goto out_free;
 
@@ -531,18 +530,18 @@ int main(int argc, char * const argv[])
 			goto out_free;
 		}
 
-		if (img) {
-			fd = open(img, O_RDONLY);
+		if (vi[i].image_file) {
+			fd = open(vi[i].image_file, O_RDONLY);
 			if (fd == -1) {
 				err = fd;
-				sys_errmsg("cannot open \"%s\"", img);
+				sys_errmsg("cannot open \"%s\"", vi[i].image_file);
 				goto out_free;
 			}
 
 			verbose(args.verbose, "writing volume %d", vi[i].id);
-			verbose(args.verbose, "image file: %s", img);
+			verbose(args.verbose, "image file: %s", vi[i].image_file);
 
-			err = ubigen_write_volume(&ui, &vi[i], args.ec, st.st_size, fd, args.out_fd);
+			err = ubigen_write_volume(&ui, &vi[i], args.ec, fd, args.out_fd);
 			close(fd);
 			if (err) {
 				errmsg("cannot write volume for section \"%s\"", sname);
-- 
2.13.6

[PATCH 4/5] libubigen: Maintain a eraseblock association table

[Richard Weinberger]

The EBA is needed later for preseeded fastmap support.

Signed-off-by: Richard Weinberger <richard@nod.at>
---
 include/libubigen.h |  1 +
 lib/libubigen.c     | 14 ++++++++++++++
 ubi-utils/ubinize.c |  2 ++
 3 files changed, 17 insertions(+)

diff --git a/include/libubigen.h b/include/libubigen.h
index 8084d0ecddb2..6073a2d72e05 100644
--- a/include/libubigen.h
+++ b/include/libubigen.h
@@ -92,6 +92,7 @@ struct ubigen_vol_info
 	uint8_t flags;
 	const char *image_file;
 	size_t image_file_len;
+	int *eba;
 };
 
 /**
diff --git a/lib/libubigen.c b/lib/libubigen.c
index 900c984229fb..f509d4d072bd 100644
--- a/lib/libubigen.c
+++ b/lib/libubigen.c
@@ -176,6 +176,10 @@ int ubigen_write_volume(const struct ubigen_info *ui,
 	long long bytes = vi->image_file_len;
 	char *inbuf, *outbuf;
 
+	for (lnum = 0; lnum < vi->used_ebs; lnum++)
+		vi->eba[lnum] = -1;
+
+
 	if (vi->id >= ui->max_volumes) {
 		errmsg("too high volume id %d, max. volumes is %d",
 		       vi->id, ui->max_volumes);
@@ -203,6 +207,7 @@ int ubigen_write_volume(const struct ubigen_info *ui,
 	memset(outbuf, 0xFF, ui->data_offs);
 	ubigen_init_ec_hdr(ui, (struct ubi_ec_hdr *)outbuf, ec);
 
+	lnum = 0;
 	while (bytes) {
 		int l;
 		struct ubi_vid_hdr *vid_hdr;
@@ -229,6 +234,12 @@ int ubigen_write_volume(const struct ubigen_info *ui,
 		memset(outbuf + ui->data_offs + len, 0xFF,
 		       ui->peb_size - ui->data_offs - len);
 
+		vi->eba[lnum] = lseek(out, 0, SEEK_CUR) / ui->peb_size;
+		if (vi->eba[lnum] == -1) {
+			sys_errmsg("cannot get offset of output file");
+			goto out_free1;
+		}
+
 		if (write(out, outbuf, ui->peb_size) != ui->peb_size) {
 			sys_errmsg("cannot write %d bytes to the output file", ui->peb_size);
 			goto out_free1;
@@ -285,6 +296,7 @@ int ubigen_write_layout_vol(const struct ubigen_info *ui, int peb1, int peb2,
 		sys_errmsg("cannot seek output file");
 		goto out_free;
 	}
+	vi->eba[0] = peb1;
 
 	ubigen_init_ec_hdr(ui, (struct ubi_ec_hdr *)outbuf, ec1);
 	ubigen_init_vid_hdr(ui, &vi, vid_hdr, 0, NULL, 0);
@@ -299,6 +311,8 @@ int ubigen_write_layout_vol(const struct ubigen_info *ui, int peb1, int peb2,
 		sys_errmsg("cannot seek output file");
 		goto out_free;
 	}
+	vi->eba[1] = peb2;
+
 	ubigen_init_ec_hdr(ui, (struct ubi_ec_hdr *)outbuf, ec2);
 	ubigen_init_vid_hdr(ui, &vi, vid_hdr, 1, NULL, 0);
 	ret = write(fd, outbuf, ui->peb_size);
diff --git a/ubi-utils/ubinize.c b/ubi-utils/ubinize.c
index bffb66ebcbd9..62ddd7b85296 100644
--- a/ubi-utils/ubinize.c
+++ b/ubi-utils/ubinize.c
@@ -398,6 +398,7 @@ static int read_section(const struct ubigen_info *ui, const char *sname,
 	else
 		vi->used_ebs = (st.st_size + vi->usable_leb_size - 1) / vi->usable_leb_size;
 	vi->compat = 0;
+	vi->eba = xcalloc(vi->used_ebs, sizeof(int));
 	return 0;
 }
 
@@ -563,6 +564,7 @@ int main(int argc, char * const argv[])
 
 	verbose(args.verbose, "done");
 
+	free(vi->eba);
 	free(vi);
 	iniparser_freedict(args.dict);
 	free(vtbl);
-- 
2.13.6

[PATCH 5/5] ubinize: Add fastmap support

[Richard Weinberger]

Using the "-F" parameter, ubinize can now create ubinized images
with a preseeded fastmap on it.
You need a recent kernel with UBI_FM_SB_PRESEEDED_FLG support to
attach from such a fastmap.

Signed-off-by: Richard Weinberger <richard@nod.at>
---
 include/libubigen.h   |  17 +++-
 lib/libubigen.c       | 224 ++++++++++++++++++++++++++++++++++++++++++++++----
 ubi-utils/ubiformat.c |   6 +-
 ubi-utils/ubinize.c   |  76 +++++++++++++----
 4 files changed, 283 insertions(+), 40 deletions(-)

diff --git a/include/libubigen.h b/include/libubigen.h
index 6073a2d72e05..7b50d903300c 100644
--- a/include/libubigen.h
+++ b/include/libubigen.h
@@ -110,6 +110,10 @@ void ubigen_info_init(struct ubigen_info *ui, int peb_size, int min_io_size,
 		      int subpage_size, int vid_hdr_offs, int ubi_ver,
 		      uint32_t image_seq);
 
+int ubigen_fastmap_size(struct ubigen_info *ui,
+			struct ubigen_vol_info *vols[], int vol_count,
+			struct ubigen_vol_info *layout_vol);
+
 /**
  * ubigen_create_empty_vtbl - creates empty volume table.
  * @ui: libubigen information
@@ -171,13 +175,18 @@ int ubigen_add_volume(const struct ubigen_info *ui,
  * writes the UBI volume to the output file @out. Returns zero on success and
  * %-1 on failure.
  */
-int ubigen_write_volume(const struct ubigen_info *ui,
+int ubigen_write_volume(struct ubigen_info *ui,
 			const struct ubigen_vol_info *vi, long long ec,
 			int in, int out);
 
+int ubigen_write_fastmap(struct ubigen_info *ui,
+			 struct ubigen_vol_info *layout_vi,
+			 int vol_count, struct ubigen_vol_info *vols[],
+			 int ec, int out_fd);
 /**
  * ubigen_write_layout_vol - write UBI layout volume
  * @ui: libubigen information
+ * @vi: volume information
  * @peb1: physical eraseblock number to write the first volume table copy
  * @peb2: physical eraseblock number to write the second volume table copy
  * @ec1: erase counter value for @peb1
@@ -188,10 +197,12 @@ int ubigen_write_volume(const struct ubigen_info *ui,
  * This function creates the UBI layout volume which contains 2 copies of the
  * volume table. Returns zero in case of success and %-1 in case of failure.
  */
-int ubigen_write_layout_vol(const struct ubigen_info *ui, int peb1, int peb2,
-			    long long ec1, long long ec2,
+int ubigen_write_layout_vol(struct ubigen_info *ui, struct ubigen_vol_info *vi,
+			    int peb1, int peb2, long long ec1, long long ec2,
 			    struct ubi_vtbl_record *vtbl, int fd);
 
+struct ubigen_vol_info *ubigen_init_layout_vi(const struct ubigen_info *ui);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/lib/libubigen.c b/lib/libubigen.c
index f509d4d072bd..05a27e639a49 100644
--- a/lib/libubigen.c
+++ b/lib/libubigen.c
@@ -30,13 +30,31 @@
 #include <stdint.h>
 #include <unistd.h>
 #include <string.h>
+#include <sys/param.h> /* roundup() */
 
 #include <mtd/ubi-media.h>
+#include <mtd/ubi-user.h>
 #include <mtd_swab.h>
 #include <libubigen.h>
 #include <crc32.h>
 #include "common.h"
 
+static size_t calc_fastmap_size(struct ubigen_info *ui, int peb_count)
+{
+	size_t size;
+
+	size = sizeof(struct ubi_fm_sb) +
+		sizeof(struct ubi_fm_hdr) +
+		sizeof(struct ubi_fm_scan_pool) +
+		sizeof(struct ubi_fm_scan_pool) +
+		(peb_count * sizeof(struct ubi_fm_ec)) +
+		(sizeof(struct ubi_fm_eba) +
+		(peb_count * sizeof(__be32))) +
+		sizeof(struct ubi_fm_volhdr) * UBI_MAX_VOLUMES;
+
+	return roundup(size, ui->leb_size);
+}
+
 void ubigen_info_init(struct ubigen_info *ui, int peb_size, int min_io_size,
 		      int subpage_size, int vid_hdr_offs, int ubi_ver,
 		      uint32_t image_seq)
@@ -63,6 +81,19 @@ void ubigen_info_init(struct ubigen_info *ui, int peb_size, int min_io_size,
 	ui->vtbl_size = ui->max_volumes * UBI_VTBL_RECORD_SIZE;
 }
 
+int ubigen_fastmap_size(struct ubigen_info *ui,
+			struct ubigen_vol_info *vols[], int vol_count,
+			struct ubigen_vol_info *layout_vol)
+{
+	int i;
+	int vol_pebs = layout_vol->used_ebs;
+
+	for (i = 0; i < vol_count; i++)
+		vol_pebs += vols[i]->used_ebs;
+
+	return calc_fastmap_size(ui, vol_pebs + UBI_FM_MAX_BLOCKS);
+}
+
 struct ubi_vtbl_record *ubigen_create_empty_vtbl(const struct ubigen_info *ui)
 {
 	struct ubi_vtbl_record *vtbl;
@@ -168,11 +199,31 @@ void ubigen_init_vid_hdr(const struct ubigen_info *ui,
 	hdr->hdr_crc = cpu_to_be32(crc);
 }
 
-int ubigen_write_volume(const struct ubigen_info *ui,
+static void ubigen_init_fm_vid_hdr(const struct ubigen_info *ui,
+				   struct ubi_vid_hdr *hdr, int vol_id,
+				   int lnum)
+{
+	uint32_t crc;
+
+	memset(hdr, 0, sizeof(struct ubi_vid_hdr));
+
+	hdr->magic = cpu_to_be32(UBI_VID_HDR_MAGIC);
+	hdr->version = ui->ubi_ver;
+	hdr->vol_type = UBI_VID_DYNAMIC;
+	hdr->vol_id = cpu_to_be32(vol_id);
+	hdr->lnum = cpu_to_be32(lnum);
+	hdr->compat = UBI_COMPAT_DELETE;
+	hdr->sqnum = cpu_to_be64(1);
+
+	crc = mtd_crc32(UBI_CRC32_INIT, hdr, UBI_VID_HDR_SIZE_CRC);
+	hdr->hdr_crc = cpu_to_be32(crc);
+}
+
+int ubigen_write_volume(struct ubigen_info *ui,
 			const struct ubigen_vol_info *vi, long long ec,
 			int in, int out)
 {
-	int len = vi->usable_leb_size, rd, lnum = 0;
+	int len = vi->usable_leb_size, rd, lnum;
 	long long bytes = vi->image_file_len;
 	char *inbuf, *outbuf;
 
@@ -259,27 +310,144 @@ out_free:
 	return -1;
 }
 
-int ubigen_write_layout_vol(const struct ubigen_info *ui, int peb1, int peb2,
-			    long long ec1, long long ec2,
+int ubigen_write_fastmap(struct ubigen_info *ui,
+			 struct ubigen_vol_info *layout_vi,
+			 int vol_count, struct ubigen_vol_info *vols[],
+			 int ec, int out_fd)
+{
+	int i, j, ret;
+	struct ubi_fm_sb *fmsb;
+	struct ubi_fm_hdr *fmh;
+	struct ubi_fm_scan_pool *fmpl, *fmpl_wl;
+	struct ubi_fm_ec *fec;
+	struct ubi_fm_volhdr *fvh;
+	struct ubi_fm_eba *feba;
+	struct ubigen_vol_info *vi;
+	size_t fm_pos = 0;
+	size_t fm_size = ubigen_fastmap_size(ui, vols, vol_count, layout_vi);
+	char *fm_raw = xcalloc(1, fm_size);
+	char *peb_buf = xmalloc(ui->peb_size);
+	int fm_pebs = fm_size / ui->leb_size;
+	int pebs_used = 0;
+
+
+	fmsb = (struct ubi_fm_sb *)fm_raw;
+	fm_pos += sizeof(*fmsb);
+
+	fmsb->magic = cpu_to_be32(UBI_FM_SB_MAGIC);
+	fmsb->version = 2;
+	fmsb->used_blocks = cpu_to_be32(fm_pebs);
+	fmsb->sqnum = 0;
+	fmsb->data_crc = 0;
+	fmsb->flags = cpu_to_be32(UBI_FM_SB_PRESEEDED_FLG);
+
+	for (i = 0; i < fm_pebs; i++) {
+		fmsb->block_loc[i] = cpu_to_be32(layout_vi->used_ebs + i);
+		fmsb->block_ec[i] = cpu_to_be32(ec);
+	}
+
+	fmh = (struct ubi_fm_hdr *)(fm_raw + fm_pos);
+	fm_pos += sizeof(*fmh);
+
+	fmh->magic = cpu_to_be32(UBI_FM_HDR_MAGIC);
+	fmh->vol_count = cpu_to_be32(vol_count + 1);
+	fmh->free_peb_count = 0;
+	fmh->scrub_peb_count = 0;
+	fmh->erase_peb_count = 0;
+	fmh->bad_peb_count = 0;
+
+	fmpl = (struct ubi_fm_scan_pool *)(fm_raw + fm_pos);
+	fm_pos += sizeof(*fmpl);
+
+	fmpl->magic = cpu_to_be32(UBI_FM_POOL_MAGIC);
+	fmpl->size = 0;
+	fmpl->max_size = 0;
+
+	fmpl_wl = (struct ubi_fm_scan_pool *)(fm_raw + fm_pos);
+	fm_pos += sizeof(*fmpl_wl);
+
+	fmpl_wl->magic = cpu_to_be32(UBI_FM_POOL_MAGIC);
+        fmpl_wl->size = 0;
+        fmpl_wl->max_size = 0;
+
+	for (i = 0; i < vol_count + 1; i++) {
+		if (i == vol_count)
+			vi = layout_vi;
+		else
+			vi = vols[i];
+
+		for (j = 0; j < vi->used_ebs; j++) {
+			fec = (struct ubi_fm_ec *)(fm_raw + fm_pos);
+
+			if (vi->eba[j] > -1) {
+				fec->pnum = cpu_to_be32(vi->eba[j]);
+				fec->ec = cpu_to_be32(ec);
+				pebs_used++;
+				fm_pos += sizeof(*fec);
+			}
+		}
+	}
+
+	fmh->used_peb_count = cpu_to_be32(pebs_used);
+
+	for (i = 0; i < vol_count + 1; i++) {
+		if (i == vol_count)
+			vi = layout_vi;
+		else
+			vi = vols[i];
+
+		fvh = (struct ubi_fm_volhdr *)(fm_raw + fm_pos);
+		fm_pos += sizeof(*fvh);
+
+		fvh->magic = cpu_to_be32(UBI_FM_VHDR_MAGIC);
+		fvh->vol_id = cpu_to_be32(vi->id);
+		fvh->vol_type = vi->type == UBI_VID_DYNAMIC ? UBI_DYNAMIC_VOLUME : UBI_STATIC_VOLUME;
+		fvh->used_ebs = cpu_to_be32(vi->used_ebs);
+		fvh->data_pad = cpu_to_be32(vi->data_pad);
+		fvh->last_eb_bytes = cpu_to_be32(vi->bytes % vi->usable_leb_size);
+
+		feba = (struct ubi_fm_eba *)(fm_raw + fm_pos);
+		fm_pos += sizeof(*feba) + (sizeof(__be32) * vi->used_ebs);
+
+		feba->magic = cpu_to_be32(UBI_FM_EBA_MAGIC);
+		feba->reserved_pebs = cpu_to_be32(vi->used_ebs);
+		for (j = 0; j < vi->used_ebs; j++)
+			feba->pnum[j] = cpu_to_be32(vi->eba[j]);
+	}
+
+	fmsb->data_crc = cpu_to_be32(mtd_crc32(UBI_CRC32_INIT, fm_raw, fm_size));
+
+	for (i = 0; i < fm_pebs; i++) {
+		int vol_id = i == 0 ? UBI_FM_SB_VOLUME_ID : UBI_FM_DATA_VOLUME_ID;
+
+		memset(peb_buf, 0xff, ui->peb_size);
+
+		ubigen_init_ec_hdr(ui, (struct ubi_ec_hdr *)peb_buf, ec);
+		ubigen_init_fm_vid_hdr(ui, (struct ubi_vid_hdr *)(peb_buf + ui->vid_hdr_offs), vol_id, i);
+		memcpy(peb_buf + ui->data_offs, fm_raw + ui->leb_size * i, ui->leb_size);
+		ret = write(out_fd, peb_buf, ui->peb_size);
+		if (ret != ui->peb_size) {
+			sys_errmsg("cannot write %d bytes", ui->peb_size);
+			goto out;
+		}
+	}
+
+	ret = 0;
+out:
+	free(fm_raw);
+	free(peb_buf);
+	return ret;
+}
+
+int ubigen_write_layout_vol(struct ubigen_info *ui, struct ubigen_vol_info *vi,
+			    int peb1, int peb2, long long ec1, long long ec2,
 			    struct ubi_vtbl_record *vtbl, int fd)
 {
 	int ret;
-	struct ubigen_vol_info vi;
 	char *outbuf;
 	struct ubi_vid_hdr *vid_hdr;
 	off_t seek;
 
-	vi.bytes = ui->leb_size * UBI_LAYOUT_VOLUME_EBS;
-	vi.id = UBI_LAYOUT_VOLUME_ID;
-	vi.alignment = UBI_LAYOUT_VOLUME_ALIGN;
-	vi.data_pad = ui->leb_size % UBI_LAYOUT_VOLUME_ALIGN;
-	vi.usable_leb_size = ui->leb_size - vi.data_pad;
-	vi.data_pad = ui->leb_size - vi.usable_leb_size;
-	vi.type = UBI_LAYOUT_VOLUME_TYPE;
-	vi.name = UBI_LAYOUT_VOLUME_NAME;
-	vi.name_len = strlen(UBI_LAYOUT_VOLUME_NAME);
-	vi.compat = UBI_LAYOUT_VOLUME_COMPAT;
-
 	outbuf = malloc(ui->peb_size);
 	if (!outbuf)
 		return sys_errmsg("failed to allocate %d bytes",
@@ -299,7 +467,7 @@ int ubigen_write_layout_vol(const struct ubigen_info *ui, int peb1, int peb2,
 	vi->eba[0] = peb1;
 
 	ubigen_init_ec_hdr(ui, (struct ubi_ec_hdr *)outbuf, ec1);
-	ubigen_init_vid_hdr(ui, &vi, vid_hdr, 0, NULL, 0);
+	ubigen_init_vid_hdr(ui, vi, vid_hdr, 0, NULL, 0);
 	ret = write(fd, outbuf, ui->peb_size);
 	if (ret != ui->peb_size) {
 		sys_errmsg("cannot write %d bytes", ui->peb_size);
@@ -314,7 +482,7 @@ int ubigen_write_layout_vol(const struct ubigen_info *ui, int peb1, int peb2,
 	vi->eba[1] = peb2;
 
 	ubigen_init_ec_hdr(ui, (struct ubi_ec_hdr *)outbuf, ec2);
-	ubigen_init_vid_hdr(ui, &vi, vid_hdr, 1, NULL, 0);
+	ubigen_init_vid_hdr(ui, vi, vid_hdr, 1, NULL, 0);
 	ret = write(fd, outbuf, ui->peb_size);
 	if (ret != ui->peb_size) {
 		sys_errmsg("cannot write %d bytes", ui->peb_size);
@@ -328,3 +496,23 @@ out_free:
 	free(outbuf);
 	return -1;
 }
+
+struct ubigen_vol_info *ubigen_init_layout_vi(const struct ubigen_info *ui)
+{
+	struct ubigen_vol_info *vi = xmalloc(sizeof(*vi));
+
+	vi->bytes = ui->leb_size * UBI_LAYOUT_VOLUME_EBS;
+	vi->id = UBI_LAYOUT_VOLUME_ID;
+	vi->alignment = UBI_LAYOUT_VOLUME_ALIGN;
+	vi->data_pad = ui->leb_size % UBI_LAYOUT_VOLUME_ALIGN;
+	vi->usable_leb_size = ui->leb_size - vi->data_pad;
+	vi->data_pad = ui->leb_size - vi->usable_leb_size;
+	vi->type = UBI_LAYOUT_VOLUME_TYPE;
+	vi->name = UBI_LAYOUT_VOLUME_NAME;
+	vi->name_len = strlen(UBI_LAYOUT_VOLUME_NAME);
+	vi->compat = UBI_LAYOUT_VOLUME_COMPAT;
+	vi->used_ebs = 2;
+	vi->eba = xcalloc(vi->used_ebs, sizeof(int));
+
+	return vi;
+}
diff --git a/ubi-utils/ubiformat.c b/ubi-utils/ubiformat.c
index c38b9b4abb16..a333246ecf77 100644
--- a/ubi-utils/ubiformat.c
+++ b/ubi-utils/ubiformat.c
@@ -553,7 +553,7 @@ out_close:
 }
 
 static int format(libmtd_t libmtd, const struct mtd_dev_info *mtd,
-		  const struct ubigen_info *ui, struct ubi_scan_info *si,
+		  struct ubigen_info *ui, struct ubi_scan_info *si,
 		  int start_eb, int novtbl)
 {
 	int eb, err, write_size;
@@ -655,6 +655,8 @@ static int format(libmtd_t libmtd, const struct mtd_dev_info *mtd,
 		printf("\n");
 
 	if (!novtbl) {
+		struct ubigen_vol_info *layout_vi = ubigen_init_layout_vi(ui);
+
 		if (eb1 == -1 || eb2 == -1) {
 			errmsg("no eraseblocks for volume table");
 			goto out_free;
@@ -665,7 +667,7 @@ static int format(libmtd_t libmtd, const struct mtd_dev_info *mtd,
 		if (!vtbl)
 			goto out_free;
 
-		err = ubigen_write_layout_vol(ui, eb1, eb2, ec1,  ec2, vtbl,
+		err = ubigen_write_layout_vol(ui, layout_vi, eb1, eb2, ec1,  ec2, vtbl,
 					      args.node_fd);
 		free(vtbl);
 		if (err) {
diff --git a/ubi-utils/ubinize.c b/ubi-utils/ubinize.c
index 62ddd7b85296..08048eceb33a 100644
--- a/ubi-utils/ubinize.c
+++ b/ubi-utils/ubinize.c
@@ -82,6 +82,7 @@ static const struct option long_options[] = {
 	{ .name = "verbose",        .has_arg = 0, .flag = NULL, .val = 'v' },
 	{ .name = "help",           .has_arg = 0, .flag = NULL, .val = 'h' },
 	{ .name = "version",        .has_arg = 0, .flag = NULL, .val = 'V' },
+	{ .name = "fastmap",        .has_arg = 0, .flag = NULL, .val = 'F' },
 	{ NULL, 0, NULL, 0}
 };
 
@@ -97,6 +98,7 @@ struct args {
 	int ubi_ver;
 	uint32_t image_seq;
 	int verbose;
+	int fastmap;
 	dictionary *dict;
 };
 
@@ -116,7 +118,7 @@ static int parse_opt(int argc, char * const argv[])
 		int key, error = 0;
 		unsigned long int image_seq;
 
-		key = getopt_long(argc, argv, "o:p:m:s:O:e:x:Q:vhV", long_options, NULL);
+		key = getopt_long(argc, argv, "o:p:m:s:O:e:x:Q:vhVF", long_options, NULL);
 		if (key == -1)
 			break;
 
@@ -180,6 +182,10 @@ static int parse_opt(int argc, char * const argv[])
 			args.verbose = 1;
 			break;
 
+		case 'F':
+			args.fastmap = 1;
+			break;
+
 		case 'h':
 			fputs(usage, stdout);
 			fputs(optionsstr, stdout);
@@ -408,6 +414,7 @@ int main(int argc, char * const argv[])
 	struct ubigen_info ui;
 	struct ubi_vtbl_record *vtbl;
 	struct ubigen_vol_info *vi;
+	struct ubigen_vol_info *layout_vi;
 	off_t seek;
 
 	err = parse_opt(argc, argv);
@@ -465,20 +472,11 @@ int main(int argc, char * const argv[])
 		goto out_dict;
 	}
 
-	/*
-	 * Skip 2 PEBs at the beginning of the file for the volume table which
-	 * will be written later.
-	 */
-	seek = ui.peb_size * 2;
-	if (lseek(args.out_fd, seek, SEEK_SET) != seek) {
-		err = -1;
-		sys_errmsg("cannot seek file \"%s\"", args.f_out);
-		goto out_free;
-	}
+	layout_vi = ubigen_init_layout_vi(&ui);
 
 	for (i = 0; i < sects; i++) {
 		const char *sname = iniparser_getsecname(args.dict, i);
-		int fd, j;
+		int j;
 
 		if (!sname) {
 			err = -1;
@@ -531,8 +529,33 @@ int main(int argc, char * const argv[])
 			goto out_free;
 		}
 
+		if (args.verbose)
+			printf("\n");
+	}
+
+	/*
+	 * Skip 2 PEBs at the beginning of the file for the volume table which
+	 * will be written later.
+	 */
+	seek = 2;
+
+	/*
+	 * If Fastmap is enabled we need also some PEBs for it at the beginning.
+	 */
+	if (args.fastmap)
+		seek += ubigen_fastmap_size(&ui, &vi, sects, layout_vi) / ui.leb_size;
+
+	seek *= ui.peb_size;
+
+	if (lseek(args.out_fd, seek, SEEK_SET) != seek) {
+		err = -1;
+		sys_errmsg("cannot seek file \"%s\"", args.f_out);
+		goto out_free;
+	}
+
+	for (i = 0; i < sects; i++) {
 		if (vi[i].image_file) {
-			fd = open(vi[i].image_file, O_RDONLY);
+			int fd = open(vi[i].image_file, O_RDONLY);
 			if (fd == -1) {
 				err = fd;
 				sys_errmsg("cannot open \"%s\"", vi[i].image_file);
@@ -545,27 +568,44 @@ int main(int argc, char * const argv[])
 			err = ubigen_write_volume(&ui, &vi[i], args.ec, fd, args.out_fd);
 			close(fd);
 			if (err) {
-				errmsg("cannot write volume for section \"%s\"", sname);
+				errmsg("cannot write volume \"%s\"", vi[i].name);
 				goto out_free;
 			}
 		}
 
-		if (args.verbose)
-			printf("\n");
 	}
 
 	verbose(args.verbose, "writing layout volume");
 
-	err = ubigen_write_layout_vol(&ui, 0, 1, args.ec, args.ec, vtbl, args.out_fd);
+	err = ubigen_write_layout_vol(&ui, layout_vi, 0, 1, args.ec, args.ec, vtbl, args.out_fd);
 	if (err) {
 		errmsg("cannot write layout volume");
 		goto out_free;
 	}
 
+	if (args.fastmap) {
+		verbose(args.verbose, "writing fastmap");
+
+		if (lseek(args.out_fd, 2 * ui.peb_size, SEEK_SET) != 2 * ui.peb_size) {
+			err = -1;
+			sys_errmsg("cannot seek file \"%s\"", args.f_out);
+			goto out_free;
+		}
+
+		err = ubigen_write_fastmap(&ui, layout_vi, sects, &vi, args.ec, args.out_fd);
+		if (err) {
+			errmsg("cannot write fastmap");
+			goto out_free;
+		}
+	}
+
+
 	verbose(args.verbose, "done");
 
 	free(vi->eba);
 	free(vi);
+	free(layout_vi->eba);
+	free(layout_vi);
 	iniparser_freedict(args.dict);
 	free(vtbl);
 	close(args.out_fd);
@@ -573,6 +613,8 @@ int main(int argc, char * const argv[])
 
 out_free:
 	free(vi);
+	free(layout_vi);
+	free(layout_vi->eba);
 out_dict:
 	iniparser_freedict(args.dict);
 out_vtbl:
-- 
2.13.6

Re: [PATCH 4/5] libubigen: Maintain a eraseblock association table

[David Oberhollenzer]

On 05/14/2018 01:25 PM, Richard Weinberger wrote:
> @@ -229,6 +234,12 @@ int ubigen_write_volume(const struct ubigen_info *ui,
>  		memset(outbuf + ui->data_offs + len, 0xFF,
>  		       ui->peb_size - ui->data_offs - len);
>  
> +		vi->eba[lnum] = lseek(out, 0, SEEK_CUR) / ui->peb_size;
> +		if (vi->eba[lnum] == -1) {
> +			sys_errmsg("cannot get offset of output file");
> +			goto out_free1;
> +		}
> +
>  		if (write(out, outbuf, ui->peb_size) != ui->peb_size) {
>  			sys_errmsg("cannot write %d bytes to the output file", ui->peb_size);
>  			goto out_free1;

Wouldn't that division swallow errors? If I interpret the C99 draft correctly, dividing
an lseek return value of -1 by ui->peb_size > 1 should result in 0.

Thanks,

David

Re: [PATCH 4/5] libubigen: Maintain a eraseblock association table

[Richard Weinberger]

Am Dienstag, 15. Mai 2018, 21:58:01 CEST schrieb David Oberhollenzer:
> On 05/14/2018 01:25 PM, Richard Weinberger wrote:
> > @@ -229,6 +234,12 @@ int ubigen_write_volume(const struct ubigen_info *ui,
> >  		memset(outbuf + ui->data_offs + len, 0xFF,
> >  		       ui->peb_size - ui->data_offs - len);
> >  
> > +		vi->eba[lnum] = lseek(out, 0, SEEK_CUR) / ui->peb_size;
> > +		if (vi->eba[lnum] == -1) {
> > +			sys_errmsg("cannot get offset of output file");
> > +			goto out_free1;
> > +		}
> > +
> >  		if (write(out, outbuf, ui->peb_size) != ui->peb_size) {
> >  			sys_errmsg("cannot write %d bytes to the output file", ui->peb_size);
> >  			goto out_free1;
> 
> Wouldn't that division swallow errors? If I interpret the C99 draft correctly, dividing
> an lseek return value of -1 by ui->peb_size > 1 should result in 0.

Yep, that needs fixing.

Thanks,
//richard