[Qemu-devel] [PATCH v9] qemu-io: add pattern file for write command

[Qemu-devel] [PATCH v9] qemu-io: add pattern file for write command

[Denis Plotnikov]

The patch allows to provide a pattern file for write
command. There was no similar ability before.

Signed-off-by: Denis Plotnikov <dplotnikov@virtuozzo.com>
---
v9:
  * replace flag cast to int with bool [Eric]
  * fix the error message [Eric]
  * use qemu_io_free instead of qemu_vfree [Eric]
  * add function description [Eric]

v8: fix according to Max's comments
  * get rid of unnecessary buffer for the pattern
  * buffer allocation just in bytes
  * take into account the missalign offset
  * don't copy file name
  * changed char* to const char* in input params

v7:
  * fix variable naming
  * make code more readable
  * extend help for write command

v6:
  * the pattern file is read once to reduce io

v5:
  * file name initiated with null to make compilers happy

v4:
  * missing signed-off clause added

v3:
  * missing file closing added
  * exclusive flags processing changed
  * buffer void* converted to char* to fix pointer arithmetics
  * file reading error processing added
---
 qemu-io-cmds.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 91 insertions(+), 6 deletions(-)

diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c
index 09750a23ce..f7bdfe673b 100644
--- a/qemu-io-cmds.c
+++ b/qemu-io-cmds.c
@@ -351,6 +351,77 @@ static void qemu_io_free(void *p)
     qemu_vfree(p);
 }
 
+/*
+ * qemu_io_alloc_from_file()
+ *
+ * Allocates the buffer and populates it with the content of the given file
+ * up to @len bytes. If the file length is less then @len, then the buffer
+ * is populated with then file content cyclically.
+ *
+ * @blk - the block backend where the buffer content is going to be written to
+ * @len - the buffer length
+ * @file_name - the file to copy the content from
+ *
+ * Returns: the buffer pointer on success
+ *          NULL on error
+ */
+static void *qemu_io_alloc_from_file(BlockBackend *blk, size_t len,
+                                     const char *file_name)
+{
+    char *buf, *buf_origin;
+    FILE *f = fopen(file_name, "r");
+    int pattern_len;
+
+    if (!f) {
+        perror(file_name);
+        return NULL;
+    }
+
+    if (qemuio_misalign) {
+        len += MISALIGN_OFFSET;
+    }
+
+    buf_origin = buf = blk_blockalign(blk, len);
+
+    if (qemuio_misalign) {
+        buf_origin += MISALIGN_OFFSET;
+    }
+
+    pattern_len = fread(buf_origin, 1, len, f);
+
+    if (ferror(f)) {
+        perror(file_name);
+        goto error;
+    }
+
+    if (pattern_len == 0) {
+        fprintf(stderr, "%s: file is empty\n", file_name);
+        goto error;
+    }
+
+    fclose(f);
+
+    if (len > pattern_len) {
+        len -= pattern_len;
+        buf += pattern_len;
+
+        while (len > 0) {
+            size_t len_to_copy = MIN(pattern_len, len);
+
+            memcpy(buf, buf_origin, len_to_copy);
+
+            len -= len_to_copy;
+            buf += len_to_copy;
+        }
+    }
+
+    return buf_origin;
+
+error:
+    qemu_io_free(buf_origin);
+    return NULL;
+}
+
 static void dump_buffer(const void *buffer, int64_t offset, int64_t len)
 {
     uint64_t i;
@@ -949,6 +1020,7 @@ static void write_help(void)
 " -n, -- with -z, don't allow slow fallback\n"
 " -p, -- ignored for backwards compatibility\n"
 " -P, -- use different pattern to fill file\n"
+" -s, -- use a pattern file to fill the write buffer\n"
 " -C, -- report statistics in a machine parsable format\n"
 " -q, -- quiet mode, do not show I/O statistics\n"
 " -u, -- with -z, allow unmapping\n"
@@ -965,7 +1037,7 @@ static const cmdinfo_t write_cmd = {
     .perm       = BLK_PERM_WRITE,
     .argmin     = 2,
     .argmax     = -1,
-    .args       = "[-bcCfnquz] [-P pattern] off len",
+    .args       = "[-bcCfnquz] [-P pattern | -s source_file] off len",
     .oneline    = "writes a number of bytes at a specified offset",
     .help       = write_help,
 };
@@ -974,7 +1046,7 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
 {
     struct timeval t1, t2;
     bool Cflag = false, qflag = false, bflag = false;
-    bool Pflag = false, zflag = false, cflag = false;
+    bool Pflag = false, zflag = false, cflag = false, sflag = false;
     int flags = 0;
     int c, cnt, ret;
     char *buf = NULL;
@@ -983,8 +1055,9 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
     /* Some compilers get confused and warn if this is not initialized.  */
     int64_t total = 0;
     int pattern = 0xcd;
+    const char *file_name = NULL;
 
-    while ((c = getopt(argc, argv, "bcCfnpP:quz")) != -1) {
+    while ((c = getopt(argc, argv, "bcCfnpP:quzs:")) != -1) {
         switch (c) {
         case 'b':
             bflag = true;
@@ -1020,6 +1093,10 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
         case 'z':
             zflag = true;
             break;
+        case 's':
+            sflag = true;
+            file_name = optarg;
+            break;
         default:
             qemuio_command_usage(&write_cmd);
             return -EINVAL;
@@ -1051,8 +1128,9 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
         return -EINVAL;
     }
 
-    if (zflag && Pflag) {
-        printf("-z and -P cannot be specified at the same time\n");
+    if ((bool)zflag + (bool)Pflag + (bool)sflag > 1) {
+        printf("Only one of -z, -P, and -s "
+               "can be specified at the same time\n");
         return -EINVAL;
     }
 
@@ -1088,7 +1166,14 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
     }
 
     if (!zflag) {
-        buf = qemu_io_alloc(blk, count, pattern);
+        if (sflag) {
+            buf = qemu_io_alloc_from_file(blk, count, file_name);
+            if (!buf) {
+                return -EINVAL;
+            }
+        } else {
+            buf = qemu_io_alloc(blk, count, pattern);
+        }
     }
 
     gettimeofday(&t1, NULL);
-- 
2.17.0

Re: [Qemu-devel] [PATCH v9] qemu-io: add pattern file for write command

[Max Reitz]

[-- Attachment #1.1: Type: text/plain, Size: 4799 bytes --]

On 19.08.19 18:18, Denis Plotnikov wrote:
> The patch allows to provide a pattern file for write
> command. There was no similar ability before.
> 
> Signed-off-by: Denis Plotnikov <dplotnikov@virtuozzo.com>
> ---
> v9:
>   * replace flag cast to int with bool [Eric]
>   * fix the error message [Eric]
>   * use qemu_io_free instead of qemu_vfree [Eric]
>   * add function description [Eric]
> 
> v8: fix according to Max's comments
>   * get rid of unnecessary buffer for the pattern
>   * buffer allocation just in bytes
>   * take into account the missalign offset
>   * don't copy file name
>   * changed char* to const char* in input params
> 
> v7:
>   * fix variable naming
>   * make code more readable
>   * extend help for write command
> 
> v6:
>   * the pattern file is read once to reduce io
> 
> v5:
>   * file name initiated with null to make compilers happy
> 
> v4:
>   * missing signed-off clause added
> 
> v3:
>   * missing file closing added
>   * exclusive flags processing changed
>   * buffer void* converted to char* to fix pointer arithmetics
>   * file reading error processing added
> ---
>  qemu-io-cmds.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++----
>  1 file changed, 91 insertions(+), 6 deletions(-)
> 
> diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c
> index 09750a23ce..f7bdfe673b 100644
> --- a/qemu-io-cmds.c
> +++ b/qemu-io-cmds.c
> @@ -351,6 +351,77 @@ static void qemu_io_free(void *p)
>      qemu_vfree(p);
>  }
>  
> +/*
> + * qemu_io_alloc_from_file()
> + *
> + * Allocates the buffer and populates it with the content of the given file
> + * up to @len bytes. If the file length is less then @len, then the buffer

s/then/than/ (the first one)

> + * is populated with then file content cyclically.

s/then/the/

> + *
> + * @blk - the block backend where the buffer content is going to be written to
> + * @len - the buffer length
> + * @file_name - the file to copy the content from

Probably better s/copy/read/

> + *
> + * Returns: the buffer pointer on success
> + *          NULL on error
> + */
> +static void *qemu_io_alloc_from_file(BlockBackend *blk, size_t len,
> +                                     const char *file_name)
> +{
> +    char *buf, *buf_origin;
> +    FILE *f = fopen(file_name, "r");
> +    int pattern_len;
> +
> +    if (!f) {
> +        perror(file_name);
> +        return NULL;
> +    }
> +
> +    if (qemuio_misalign) {
> +        len += MISALIGN_OFFSET;
> +    }
> +
> +    buf_origin = buf = blk_blockalign(blk, len);
> +
> +    if (qemuio_misalign) {
> +        buf_origin += MISALIGN_OFFSET;
> +    }
> +
> +    pattern_len = fread(buf_origin, 1, len, f);

Pulling the misalignment up here has more effects than just requiring
qemu_io_free() rather than qemu_vfree().

First, it breaks this fread(), because @len is the length of the buffer
in total, so this here is a buffer overflow.

> +
> +    if (ferror(f)) {
> +        perror(file_name);
> +        goto error;
> +    }
> +
> +    if (pattern_len == 0) {
> +        fprintf(stderr, "%s: file is empty\n", file_name);
> +        goto error;
> +    }
> +
> +    fclose(f);
> +
> +    if (len > pattern_len) {
> +        len -= pattern_len;
> +        buf += pattern_len;
> +
> +        while (len > 0) {
> +            size_t len_to_copy = MIN(pattern_len, len);
> +
> +            memcpy(buf, buf_origin, len_to_copy);

Second, it breaks this memcpy(), because now [buf, buf + len_to_copy)
and [buf_origin, buf_origin + len_to_copy) may overlap.

I think the solution would be (1) to add MISALIGN_OFFSET not only to
@buf_origin, but to @buf also, and (2) to reduce len by MISALIGN_OFFSET.


So all in all, I think both issues should be fixed if you add
“buf += MISALIGN_OFFSET” and “len -= MISALIGN_OFFSET” to the second
“if (qemuio_misalign)” block.  I think.

> +
> +            len -= len_to_copy;
> +            buf += len_to_copy;
> +        }
> +    }
> +
> +    return buf_origin;
> +
> +error:
> +    qemu_io_free(buf_origin);
> +    return NULL;
> +}
> +
>  static void dump_buffer(const void *buffer, int64_t offset, int64_t len)
>  {
>      uint64_t i;

[...]

> @@ -1051,8 +1128,9 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
>          return -EINVAL;
>      }
>  
> -    if (zflag && Pflag) {
> -        printf("-z and -P cannot be specified at the same time\n");
> +    if ((bool)zflag + (bool)Pflag + (bool)sflag > 1) {

Eric’s point was that you don’t need to cast at all.

Max

> +        printf("Only one of -z, -P, and -s "
> +               "can be specified at the same time\n");
>          return -EINVAL;
>      }
>  


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 488 bytes --]

Re: [Qemu-devel] [PATCH v9] qemu-io: add pattern file for write command

[Denis Plotnikov]

On Aug 20 2019, at 4:35 pm, Max Reitz <mreitz@redhat.com> wrote:
On 19.08.19 18:18, Denis Plotnikov wrote:
The patch allows to provide a pattern file for write
command. There was no similar ability before.

Signed-off-by: Denis Plotnikov <dplotnikov@virtuozzo.com>
---
v9:
* replace flag cast to int with bool [Eric]
* fix the error message [Eric]
* use qemu_io_free instead of qemu_vfree [Eric]
* add function description [Eric]

v8: fix according to Max's comments
* get rid of unnecessary buffer for the pattern
* buffer allocation just in bytes
* take into account the missalign offset
* don't copy file name
* changed char* to const char* in input params

v7:
* fix variable naming
* make code more readable
* extend help for write command

v6:
* the pattern file is read once to reduce io

v5:
* file name initiated with null to make compilers happy

v4:
* missing signed-off clause added

v3:
* missing file closing added
* exclusive flags processing changed
* buffer void* converted to char* to fix pointer arithmetics
* file reading error processing added
---
qemu-io-cmds.c | 97 ++++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 91 insertions(+), 6 deletions(-)

diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c
index 09750a23ce..f7bdfe673b 100644
--- a/qemu-io-cmds.c
+++ b/qemu-io-cmds.c
@@ -351,6 +351,77 @@ static void qemu_io_free(void *p)
qemu_vfree(p);
}

+/*
+ * qemu_io_alloc_from_file()
+ *
+ * Allocates the buffer and populates it with the content of the given file
+ * up to @len bytes. If the file length is less then @len, then the buffer

s/then/than/ (the first one)

+ * is populated with then file content cyclically.

s/then/the/

+ *
+ * @blk - the block backend where the buffer content is going to be written to
+ * @len - the buffer length
+ * @file_name - the file to copy the content from

Probably better s/copy/read/

+ *
+ * Returns: the buffer pointer on success
+ * NULL on error
+ */
+static void *qemu_io_alloc_from_file(BlockBackend *blk, size_t len,
+ const char *file_name)
+{
+ char *buf, *buf_origin;
+ FILE *f = fopen(file_name, "r");
+ int pattern_len;
+
+ if (!f) {
+ perror(file_name);
+ return NULL;
+ }
+
+ if (qemuio_misalign) {
+ len += MISALIGN_OFFSET;
+ }
+
+ buf_origin = buf = blk_blockalign(blk, len);
+
+ if (qemuio_misalign) {
+ buf_origin += MISALIGN_OFFSET;
+ }
+
+ pattern_len = fread(buf_origin, 1, len, f);

Pulling the misalignment up here has more effects than just requiring
qemu_io_free() rather than qemu_vfree().

First, it breaks this fread(), because @len is the length of the buffer
in total, so this here is a buffer overflow.

+
+ if (ferror(f)) {
+ perror(file_name);
+ goto error;
+ }
+
+ if (pattern_len == 0) {
+ fprintf(stderr, "%s: file is empty\n", file_name);
+ goto error;
+ }
+
+ fclose(f);
+
+ if (len > pattern_len) {
+ len -= pattern_len;
+ buf += pattern_len;
+
+ while (len > 0) {
+ size_t len_to_copy = MIN(pattern_len, len);
+
+ memcpy(buf, buf_origin, len_to_copy);

Second, it breaks this memcpy(), because now [buf, buf + len_to_copy)
and [buf_origin, buf_origin + len_to_copy) may overlap.

I think the solution would be (1) to add MISALIGN_OFFSET not only to
@buf_origin, but to @buf also, and (2) to reduce len by MISALIGN_OFFSET.


So all in all, I think both issues should be fixed if you add
“buf += MISALIGN_OFFSET” and “len -= MISALIGN_OFFSET” to the second
“if (qemuio_misalign)” block. I think.

Yes, thanks for pointing out

Denis

+
+ len -= len_to_copy;
+ buf += len_to_copy;
+ }
+ }
+
+ return buf_origin;
+
+error:
+ qemu_io_free(buf_origin);
+ return NULL;
+}
+
static void dump_buffer(const void *buffer, int64_t offset, int64_t len)
{
uint64_t i;

[...]

@@ -1051,8 +1128,9 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
return -EINVAL;
}

- if (zflag && Pflag) {
- printf("-z and -P cannot be specified at the same time\n");
+ if ((bool)zflag + (bool)Pflag + (bool)sflag > 1) {

Eric’s point was that you don’t need to cast at all.

Max

+ printf("Only one of -z, -P, and -s "
+ "can be specified at the same time\n");
return -EINVAL;
}