[Qemu-devel] [PATCH v2 0/2] In memory QEMUFile

[Qemu-devel] [PATCH v2 0/2] In memory QEMUFile

[Dr. David Alan Gilbert (git)]

From: "Dr. David Alan Gilbert" <dgilbert@redhat.com>

Hi,
  This patch-pair adds the QEMUSizedBuffer based in-memory QEMUFile
written by Stefan Berger and Joel Schopp.  I've made some minor
fixes (and typo cleanups) and modified the existing test-vmstate
to use it for some test cases.

  While there's nothing other than test cases using it yet, I think
it's worth going in by itself, since I'm using it in two separate
patchsets (postcopy and visitor/BER) and Sanidhya uses it in
the periodic vmstate test world.  In addition both microcheckpointing and
COLO have similar but independent implementations (although they both
have some extra-gotcha's so it might not be possible to reuse it), and
there was another implementation of the same thing in the Yabusame Postcopy
world.  Thus it seems best to put in, if only to stop people writing yet
another implementation.


Dr. David Alan Gilbert (2):
  QEMUSizedBuffer based QEMUFile
  Tests: QEMUSizedBuffer/QEMUBuffer

 include/migration/qemu-file.h |  28 +++
 include/qemu/typedefs.h       |   1 +
 qemu-file.c                   | 410 ++++++++++++++++++++++++++++++++++++++++++
 tests/Makefile                |   2 +-
 tests/test-vmstate.c          |  73 ++++----
 5 files changed, 477 insertions(+), 37 deletions(-)

-- 
v2 Comment space fixes
1.9.3

[Qemu-devel] [PATCH v2 1/2] QEMUSizedBuffer based QEMUFile

[Dr. David Alan Gilbert (git)]

From: "Dr. David Alan Gilbert" <dgilbert@redhat.com>

This is based on Stefan and Joel's patch that creates a QEMUFile that goes
to a memory buffer; from:

http://lists.gnu.org/archive/html/qemu-devel/2013-03/msg05036.html

Using the QEMUFile interface, this patch adds support functions for
operating on in-memory sized buffers that can be written to or read from.

Signed-off-by: Stefan Berger <stefanb@linux.vnet.ibm.com>
Signed-off-by: Joel Schopp <jschopp@linux.vnet.ibm.com>

For minor tweeks/rebase I've done to it:
Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
 include/migration/qemu-file.h |  28 +++
 include/qemu/typedefs.h       |   1 +
 qemu-file.c                   | 410 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 439 insertions(+)

diff --git a/include/migration/qemu-file.h b/include/migration/qemu-file.h
index c90f529..80af3ff 100644
--- a/include/migration/qemu-file.h
+++ b/include/migration/qemu-file.h
@@ -25,6 +25,8 @@
 #define QEMU_FILE_H 1
 #include "exec/cpu-common.h"
 
+#include <stdint.h>
+
 /* This function writes a chunk of data to a file at the given position.
  * The pos argument can be ignored if the file is only being used for
  * streaming.  The handler should try to write all of the data it can.
@@ -94,11 +96,21 @@ typedef struct QEMUFileOps {
     QEMURamSaveFunc *save_page;
 } QEMUFileOps;
 
+struct QEMUSizedBuffer {
+    struct iovec *iov;
+    size_t n_iov;
+    size_t size; /* total allocated size in all iov's */
+    size_t used; /* number of used bytes */
+};
+
+typedef struct QEMUSizedBuffer QEMUSizedBuffer;
+
 QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops);
 QEMUFile *qemu_fopen(const char *filename, const char *mode);
 QEMUFile *qemu_fdopen(int fd, const char *mode);
 QEMUFile *qemu_fopen_socket(int fd, const char *mode);
 QEMUFile *qemu_popen_cmd(const char *command, const char *mode);
+QEMUFile *qemu_bufopen(const char *mode, QEMUSizedBuffer *input);
 int qemu_get_fd(QEMUFile *f);
 int qemu_fclose(QEMUFile *f);
 int64_t qemu_ftell(QEMUFile *f);
@@ -111,6 +123,22 @@ void qemu_put_byte(QEMUFile *f, int v);
 void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, int size);
 bool qemu_file_mode_is_not_valid(const char *mode);
 
+QEMUSizedBuffer *qsb_create(const uint8_t *buffer, size_t len);
+QEMUSizedBuffer *qsb_clone(const QEMUSizedBuffer *);
+void qsb_free(QEMUSizedBuffer *);
+size_t qsb_set_length(QEMUSizedBuffer *qsb, size_t length);
+size_t qsb_get_length(const QEMUSizedBuffer *qsb);
+ssize_t qsb_get_buffer(const QEMUSizedBuffer *, off_t start, size_t count,
+                       uint8_t **buf);
+ssize_t qsb_write_at(QEMUSizedBuffer *qsb, const uint8_t *buf,
+                     off_t pos, size_t count);
+
+
+/*
+ * For use on files opened with qemu_bufopen
+ */
+const QEMUSizedBuffer *qemu_buf_get(QEMUFile *f);
+
 static inline void qemu_put_ubyte(QEMUFile *f, unsigned int v)
 {
     qemu_put_byte(f, (int)v);
diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h
index 5f20b0e..db1153a 100644
--- a/include/qemu/typedefs.h
+++ b/include/qemu/typedefs.h
@@ -60,6 +60,7 @@ typedef struct PCIEAERLog PCIEAERLog;
 typedef struct PCIEAERErr PCIEAERErr;
 typedef struct PCIEPort PCIEPort;
 typedef struct PCIESlot PCIESlot;
+typedef struct QEMUSizedBuffer QEMUSizedBuffer;
 typedef struct MSIMessage MSIMessage;
 typedef struct SerialState SerialState;
 typedef struct PCMCIACardState PCMCIACardState;
diff --git a/qemu-file.c b/qemu-file.c
index a8e3912..50845c9 100644
--- a/qemu-file.c
+++ b/qemu-file.c
@@ -878,3 +878,413 @@ uint64_t qemu_get_be64(QEMUFile *f)
     v |= qemu_get_be32(f);
     return v;
 }
+
+#define QSB_CHUNK_SIZE      (1 << 10)
+#define QSB_MAX_CHUNK_SIZE  (10 * QSB_CHUNK_SIZE)
+
+/**
+ * Create a QEMUSizedBuffer
+ * This type of buffer uses scatter-gather lists internally and
+ * can grow to any size. Any data array in the scatter-gather list
+ * can hold different amount of bytes.
+ *
+ * @buffer: Optional buffer to copy into the QSB
+ * @len: size of initial buffer; if @buffer is given, buffer must
+ *       hold at least len bytes
+ *
+ * Returns a pointer to a QEMUSizedBuffer
+ */
+QEMUSizedBuffer *qsb_create(const uint8_t *buffer, size_t len)
+{
+    QEMUSizedBuffer *qsb;
+    size_t alloc_len, num_chunks, i, to_copy;
+    size_t chunk_size = (len > QSB_MAX_CHUNK_SIZE)
+                        ? QSB_MAX_CHUNK_SIZE
+                        : QSB_CHUNK_SIZE;
+
+    if (len == 0) {
+        /* we want to allocate at least one chunk */
+        len = QSB_CHUNK_SIZE;
+    }
+
+    num_chunks = DIV_ROUND_UP(len, chunk_size);
+    alloc_len = num_chunks * chunk_size;
+
+    qsb = g_new0(QEMUSizedBuffer, 1);
+    qsb->iov = g_new0(struct iovec, num_chunks);
+    qsb->n_iov = num_chunks;
+
+    for (i = 0; i < num_chunks; i++) {
+        qsb->iov[i].iov_base = g_malloc0(chunk_size);
+        qsb->iov[i].iov_len = chunk_size;
+        if (buffer) {
+            to_copy = (len - qsb->used) > chunk_size
+                      ? chunk_size : (len - qsb->used);
+            memcpy(qsb->iov[i].iov_base, &buffer[qsb->used], to_copy);
+            qsb->used += to_copy;
+        }
+    }
+
+    qsb->size = alloc_len;
+
+    return qsb;
+}
+
+/**
+ * Free the QEMUSizedBuffer
+ *
+ * @qsb: The QEMUSizedBuffer to free
+ */
+void qsb_free(QEMUSizedBuffer *qsb)
+{
+    size_t i;
+
+    if (!qsb) {
+        return;
+    }
+
+    for (i = 0; i < qsb->n_iov; i++) {
+        g_free(qsb->iov[i].iov_base);
+    }
+    g_free(qsb->iov);
+    g_free(qsb);
+}
+
+/**
+ * Get the number of of used bytes in the QEMUSizedBuffer
+ *
+ * @qsb: A QEMUSizedBuffer
+ *
+ * Returns the number of bytes currently used in this buffer
+ */
+size_t qsb_get_length(const QEMUSizedBuffer *qsb)
+{
+    return qsb->used;
+}
+
+/**
+ * Set the length of the buffer; the primary usage of this
+ * function is to truncate the number of used bytes in the buffer.
+ * The size will not be extended beyond the current number of
+ * allocated bytes in the QEMUSizedBuffer.
+ *
+ * @qsb: A QEMUSizedBuffer
+ * @new_len: The new length of bytes in the buffer
+ *
+ * Returns the number of bytes the buffer was truncated or extended
+ * to.
+ */
+size_t qsb_set_length(QEMUSizedBuffer *qsb, size_t new_len)
+{
+    if (new_len <= qsb->size) {
+        qsb->used = new_len;
+    } else {
+        qsb->used = qsb->size;
+    }
+    return qsb->used;
+}
+
+/**
+ * Get the iovec that holds the data for a given position @pos.
+ *
+ * @qsb: A QEMUSizedBuffer
+ * @pos: The index of a byte in the buffer
+ * @d_off: Pointer to an offset that this function will indicate
+ *         at what position within the returned iovec the byte
+ *         is to be found
+ *
+ * Returns the index of the iovec that holds the byte at the given
+ * index @pos in the byte stream; a negative number if the iovec
+ * for the given position @pos does not exist.
+ */
+static ssize_t qsb_get_iovec(const QEMUSizedBuffer *qsb,
+                             off_t pos, off_t *d_off)
+{
+    ssize_t i;
+    off_t curr = 0;
+
+    if (pos > qsb->used) {
+        return -1;
+    }
+
+    for (i = 0; i < qsb->n_iov; i++) {
+        if (curr + qsb->iov[i].iov_len > pos) {
+            *d_off = pos - curr;
+            return i;
+        }
+        curr += qsb->iov[i].iov_len;
+    }
+    return -1;
+}
+
+/*
+ * Convert the QEMUSizedBuffer into a flat buffer.
+ *
+ * Note: If at all possible, try to avoid this function since it
+ *       may unnecessarily copy memory around.
+ *
+ * @qsb: pointer to QEMUSizedBuffer
+ * @start: offset to start at
+ * @count: number of bytes to copy
+ * @buf: a pointer to an optional buffer to write into; the pointer may
+ *       point to NULL in which case the buffer will be allocated;
+ *       if buffer is provided, it must be large enough to hold @count bytes
+ *
+ * Returns the number of bytes copied into the output buffer
+ */
+ssize_t qsb_get_buffer(const QEMUSizedBuffer *qsb, off_t start,
+                       size_t count, uint8_t **buf)
+{
+    uint8_t *buffer;
+    const struct iovec *iov;
+    size_t to_copy, all_copy;
+    ssize_t index;
+    off_t s_off;
+    off_t d_off = 0;
+    char *s;
+
+    if (start > qsb->used) {
+        return 0;
+    }
+
+    all_copy = qsb->used - start;
+    if (all_copy > count) {
+        all_copy = count;
+    } else {
+        count = all_copy;
+    }
+
+    if (*buf == NULL) {
+        *buf = g_malloc(all_copy);
+    }
+    buffer = *buf;
+
+    index = qsb_get_iovec(qsb, start, &s_off);
+    if (index < 0) {
+        return 0;
+    }
+
+    while (all_copy > 0) {
+        iov = &qsb->iov[index];
+
+        s = iov->iov_base;
+
+        to_copy = iov->iov_len - s_off;
+        if (to_copy > all_copy) {
+            to_copy = all_copy;
+        }
+        memcpy(&buffer[d_off], &s[s_off], to_copy);
+
+        d_off += to_copy;
+        all_copy -= to_copy;
+
+        s_off = 0;
+        index++;
+    }
+
+    return count;
+}
+
+/**
+ * Grow the QEMUSizedBuffer to the given size and allocated
+ * memory for it.
+ *
+ * @qsb: A QEMUSizedBuffer
+ * @new_size: The new size of the buffer
+ *
+ * Returns an error code in case of memory allocation failure
+ * or the new size of the buffer otherwise. The returned size
+ * may be greater or equal to @new_size.
+ */
+static ssize_t qsb_grow(QEMUSizedBuffer *qsb, size_t new_size)
+{
+    size_t needed_chunks, i;
+    size_t chunk_size = QSB_CHUNK_SIZE;
+
+    if (qsb->size < new_size) {
+        needed_chunks = DIV_ROUND_UP(new_size - qsb->size,
+                                     chunk_size);
+
+        qsb->iov = g_realloc_n(qsb->iov, qsb->n_iov + needed_chunks,
+                               sizeof(struct iovec));
+        if (qsb->iov == NULL) {
+            return -ENOMEM;
+        }
+
+        for (i = qsb->n_iov; i < qsb->n_iov + needed_chunks; i++) {
+            qsb->iov[i].iov_base = g_malloc0(chunk_size);
+            qsb->iov[i].iov_len = chunk_size;
+        }
+
+        qsb->n_iov += needed_chunks;
+        qsb->size += (needed_chunks * chunk_size);
+    }
+
+    return qsb->size;
+}
+
+/**
+ * Write into the QEMUSizedBuffer at a given position and a given
+ * number of bytes. This function will automatically grow the
+ * QEMUSizedBuffer.
+ *
+ * @qsb: A QEMUSizedBuffer
+ * @source: A byte array to copy data from
+ * @pos: The position withing the @qsb to write data to
+ * @size: The number of bytes to copy into the @qsb
+ *
+ * Returns an error code in case of memory allocation failure,
+ * @size otherwise.
+ */
+ssize_t qsb_write_at(QEMUSizedBuffer *qsb, const uint8_t *source,
+                     off_t pos, size_t count)
+{
+    ssize_t rc = qsb_grow(qsb, pos + count);
+    size_t to_copy;
+    size_t all_copy = count;
+    const struct iovec *iov;
+    ssize_t index;
+    char *dest;
+    off_t d_off, s_off = 0;
+
+    if (rc < 0) {
+        return rc;
+    }
+
+    if (pos + count > qsb->used) {
+        qsb->used = pos + count;
+    }
+
+    index = qsb_get_iovec(qsb, pos, &d_off);
+    if (index < 0) {
+        return 0;
+    }
+
+    while (all_copy > 0) {
+        iov = &qsb->iov[index];
+
+        dest = iov->iov_base;
+
+        to_copy = iov->iov_len - d_off;
+        if (to_copy > all_copy) {
+            to_copy = all_copy;
+        }
+
+        memcpy(&dest[d_off], &source[s_off], to_copy);
+
+        s_off += to_copy;
+        all_copy -= to_copy;
+
+        d_off = 0;
+        index++;
+    }
+
+    return count;
+}
+
+/**
+ * Create an exact copy of the given QEMUSizedBuffer.
+ *
+ * @qsb: A QEMUSizedBuffer
+ *
+ * Returns a clone of @qsb
+ */
+QEMUSizedBuffer *qsb_clone(const QEMUSizedBuffer *qsb)
+{
+    QEMUSizedBuffer *out = qsb_create(NULL, qsb_get_length(qsb));
+    size_t i;
+    off_t pos = 0;
+
+    for (i = 0; i < qsb->n_iov; i++) {
+        pos += qsb_write_at(out, qsb->iov[i].iov_base,
+                            pos, qsb->iov[i].iov_len);
+    }
+
+    return out;
+}
+
+typedef struct QEMUBuffer {
+    QEMUSizedBuffer *qsb;
+    QEMUFile *file;
+} QEMUBuffer;
+
+static int buf_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
+{
+    QEMUBuffer *s = opaque;
+    ssize_t len = qsb_get_length(s->qsb) - pos;
+
+    if (len <= 0) {
+        return 0;
+    }
+
+    if (len > size) {
+        len = size;
+    }
+    return qsb_get_buffer(s->qsb, pos, len, &buf);
+}
+
+static int buf_put_buffer(void *opaque, const uint8_t *buf,
+                          int64_t pos, int size)
+{
+    QEMUBuffer *s = opaque;
+
+    return qsb_write_at(s->qsb, buf, pos, size);
+}
+
+static int buf_close(void *opaque)
+{
+    QEMUBuffer *s = opaque;
+
+    qsb_free(s->qsb);
+
+    g_free(s);
+
+    return 0;
+}
+
+const QEMUSizedBuffer *qemu_buf_get(QEMUFile *f)
+{
+    QEMUBuffer *p;
+
+    qemu_fflush(f);
+
+    p = (QEMUBuffer *)f->opaque;
+
+    return p->qsb;
+}
+
+static const QEMUFileOps buf_read_ops = {
+    .get_buffer = buf_get_buffer,
+    .close =      buf_close
+};
+
+static const QEMUFileOps buf_write_ops = {
+    .put_buffer = buf_put_buffer,
+    .close =      buf_close
+};
+
+QEMUFile *qemu_bufopen(const char *mode, QEMUSizedBuffer *input)
+{
+    QEMUBuffer *s;
+
+    if (mode == NULL || (mode[0] != 'r' && mode[0] != 'w') || mode[1] != 0) {
+        fprintf(stderr, "qemu_bufopen: Argument validity check failed\n");
+        return NULL;
+    }
+
+    s = g_malloc0(sizeof(QEMUBuffer));
+    if (mode[0] == 'r') {
+        s->qsb = input;
+    }
+
+    if (s->qsb == NULL) {
+        s->qsb = qsb_create(NULL, 0);
+    }
+
+    if (mode[0] == 'r') {
+        s->file = qemu_fopen_ops(s, &buf_read_ops);
+    } else {
+        s->file = qemu_fopen_ops(s, &buf_write_ops);
+    }
+    return s->file;
+}
-- 
1.9.3

Re: [Qemu-devel] [PATCH v2 1/2] QEMUSizedBuffer based QEMUFile

[Eric Blake]

[-- Attachment #1: Type: text/plain, Size: 7775 bytes --]

On 08/07/2014 04:24 AM, Dr. David Alan Gilbert (git) wrote:
> From: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
> 
> This is based on Stefan and Joel's patch that creates a QEMUFile that goes
> to a memory buffer; from:
> 
> http://lists.gnu.org/archive/html/qemu-devel/2013-03/msg05036.html
> 
> Using the QEMUFile interface, this patch adds support functions for
> operating on in-memory sized buffers that can be written to or read from.
> 
> Signed-off-by: Stefan Berger <stefanb@linux.vnet.ibm.com>
> Signed-off-by: Joel Schopp <jschopp@linux.vnet.ibm.com>
> 
> For minor tweeks/rebase I've done to it:
> Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
> ---
>  include/migration/qemu-file.h |  28 +++
>  include/qemu/typedefs.h       |   1 +
>  qemu-file.c                   | 410 ++++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 439 insertions(+)
> 

>  
> +QEMUSizedBuffer *qsb_create(const uint8_t *buffer, size_t len);
> +QEMUSizedBuffer *qsb_clone(const QEMUSizedBuffer *);
> +void qsb_free(QEMUSizedBuffer *);
> +size_t qsb_set_length(QEMUSizedBuffer *qsb, size_t length);

Just on the prototype, I wonder if this should return ssize_t, to allow
for the possibility of failure...

> +/**
> + * Create a QEMUSizedBuffer
> + * This type of buffer uses scatter-gather lists internally and
> + * can grow to any size. Any data array in the scatter-gather list
> + * can hold different amount of bytes.
> + *
> + * @buffer: Optional buffer to copy into the QSB
> + * @len: size of initial buffer; if @buffer is given, buffer must
> + *       hold at least len bytes
> + *
> + * Returns a pointer to a QEMUSizedBuffer
> + */
> +QEMUSizedBuffer *qsb_create(const uint8_t *buffer, size_t len)
> +{
> +    QEMUSizedBuffer *qsb;
> +    size_t alloc_len, num_chunks, i, to_copy;
> +    size_t chunk_size = (len > QSB_MAX_CHUNK_SIZE)
> +                        ? QSB_MAX_CHUNK_SIZE
> +                        : QSB_CHUNK_SIZE;
> +
> +    if (len == 0) {
> +        /* we want to allocate at least one chunk */
> +        len = QSB_CHUNK_SIZE;
> +    }

So if I call qsb_create("", 0), len is now QSB_CHUNK_SIZE...

> +
> +    num_chunks = DIV_ROUND_UP(len, chunk_size);

...num_chunks is now 1...

> +    alloc_len = num_chunks * chunk_size;
> +
> +    qsb = g_new0(QEMUSizedBuffer, 1);
> +    qsb->iov = g_new0(struct iovec, num_chunks);
> +    qsb->n_iov = num_chunks;
> +
> +    for (i = 0; i < num_chunks; i++) {
> +        qsb->iov[i].iov_base = g_malloc0(chunk_size);
> +        qsb->iov[i].iov_len = chunk_size;
> +        if (buffer) {

...we enter the loop, and have a buffer to copy...

> +            to_copy = (len - qsb->used) > chunk_size
> +                      ? chunk_size : (len - qsb->used);
> +            memcpy(qsb->iov[i].iov_base, &buffer[qsb->used], to_copy);

...and proceed to dereference QSB_CHUNK_SIZE bytes beyond the end of the
empty string that we are converting to a buffer.  Oops.

Might be as simple as adding if (buffer) { assert(len); } before
modifying len.

> +/**
> + * Set the length of the buffer; the primary usage of this
> + * function is to truncate the number of used bytes in the buffer.
> + * The size will not be extended beyond the current number of
> + * allocated bytes in the QEMUSizedBuffer.
> + *
> + * @qsb: A QEMUSizedBuffer
> + * @new_len: The new length of bytes in the buffer
> + *
> + * Returns the number of bytes the buffer was truncated or extended
> + * to.

Confusing; I'd suggest:

Returns the resulting (possibly new) size of the buffer.  Oh, and my
earlier question appears to be answered - this function can't fail.

> +ssize_t qsb_get_buffer(const QEMUSizedBuffer *qsb, off_t start,
> +                       size_t count, uint8_t **buf)
> +{
> +    uint8_t *buffer;
> +    const struct iovec *iov;
> +    size_t to_copy, all_copy;
> +    ssize_t index;
> +    off_t s_off;
> +    off_t d_off = 0;
> +    char *s;
> +
> +    if (start > qsb->used) {
> +        return 0;
> +    }

It feels confusing that this can return 0 while leaving buf un-malloc'd.
 It's probably simpler to callers if a non-negative return value ALWAYS
guarantees that buf is valid.


> +
> +    index = qsb_get_iovec(qsb, start, &s_off);
> +    if (index < 0) {
> +        return 0;
> +    }

Wait - how is a negative value possible?  Since we already checked
against qsb->used, can't this be assert(index >= 0)?

> +static ssize_t qsb_grow(QEMUSizedBuffer *qsb, size_t new_size)
> +{
> +    size_t needed_chunks, i;
> +    size_t chunk_size = QSB_CHUNK_SIZE;
> +
> +    if (qsb->size < new_size) {
> +        needed_chunks = DIV_ROUND_UP(new_size - qsb->size,
> +                                     chunk_size);
> +

Hmm. Original creation will use chunks larger than QSB_CHUNK_SIZE (up to
the maximum chunk size), presumably for efficiency.  Should this
function likewise use larger chunks if the size to be grown is larger
than the default chunk size?

> +        qsb->iov = g_realloc_n(qsb->iov, qsb->n_iov + needed_chunks,
> +                               sizeof(struct iovec));
> +        if (qsb->iov == NULL) {

Wait. Can g_realloc_n fail? I know that g_malloc can't fail, and you
have to use g_try_malloc instead.  Do you need to try a different variant?

> +            return -ENOMEM;
> +        }
> +
> +        for (i = qsb->n_iov; i < qsb->n_iov + needed_chunks; i++) {
> +            qsb->iov[i].iov_base = g_malloc0(chunk_size);

Wait.  Why do you care about realloc'ing the iov array failing
(unlikely, as that's a relatively small allocation unlikely to be more
than a megabyte) but completely ignore the possibility of abort'ing when
g_malloc0 of a chunk size fails (much bigger, and therefore more likely
to fail if memory pressure is high)?  Either abort is always fine (no
need to ever report -ENOMEM) or you should be using a different
allocation function here.

> +/**
> + * Write into the QEMUSizedBuffer at a given position and a given
> + * number of bytes. This function will automatically grow the
> + * QEMUSizedBuffer.
> + *
> + * @qsb: A QEMUSizedBuffer
> + * @source: A byte array to copy data from
> + * @pos: The position withing the @qsb to write data to

s/withing/within/

> + * @size: The number of bytes to copy into the @qsb
> + *
> + * Returns an error code in case of memory allocation failure,
> + * @size otherwise.
> + */
> +ssize_t qsb_write_at(QEMUSizedBuffer *qsb, const uint8_t *source,
> +                     off_t pos, size_t count)
> +{
> +    ssize_t rc = qsb_grow(qsb, pos + count);
> +    size_t to_copy;
> +    size_t all_copy = count;
> +    const struct iovec *iov;
> +    ssize_t index;
> +    char *dest;
> +    off_t d_off, s_off = 0;
> +
> +    if (rc < 0) {
> +        return rc;
> +    }
> +
> +    if (pos + count > qsb->used) {
> +        qsb->used = pos + count;
> +    }
> +
> +    index = qsb_get_iovec(qsb, pos, &d_off);
> +    if (index < 0) {

Shouldn't this be an assert, because the prior qsb_grow() should have
guaranteed that we are large enough?

> +/**
> + * Create an exact copy of the given QEMUSizedBuffer.

s/exact // - it's definitely a copy representing the same contents, but
might have different boundaries for internal iovs, so omitting the word
will avoid confusion on whether that was intended.  (If you wanted an
exact copy, then I would manually allocate each iov to the right size,
and memcpy contents over from the original, instead of using qsb_create
and qsb_write_at).

-- 
Eric Blake   eblake redhat com    +1-919-301-3266
Libvirt virtualization library http://libvirt.org


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 539 bytes --]

Re: [Qemu-devel] [PATCH v2 1/2] QEMUSizedBuffer based QEMUFile

[Dr. David Alan Gilbert]

* Eric Blake (eblake@redhat.com) wrote:
> On 08/07/2014 04:24 AM, Dr. David Alan Gilbert (git) wrote:
> > From: "Dr. David Alan Gilbert" <dgilbert@redhat.com>
> > 
> > This is based on Stefan and Joel's patch that creates a QEMUFile that goes
> > to a memory buffer; from:

Hi Eric,
  I think I agree with most of your points here; and believe I've nailed
them in the v3 I've just posted.

Dave

> > 
> > http://lists.gnu.org/archive/html/qemu-devel/2013-03/msg05036.html
> > 
> > Using the QEMUFile interface, this patch adds support functions for
> > operating on in-memory sized buffers that can be written to or read from.
> > 
> > Signed-off-by: Stefan Berger <stefanb@linux.vnet.ibm.com>
> > Signed-off-by: Joel Schopp <jschopp@linux.vnet.ibm.com>
> > 
> > For minor tweeks/rebase I've done to it:
> > Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
> > ---
> >  include/migration/qemu-file.h |  28 +++
> >  include/qemu/typedefs.h       |   1 +
> >  qemu-file.c                   | 410 ++++++++++++++++++++++++++++++++++++++++++
> >  3 files changed, 439 insertions(+)
> > 
> 
> >  
> > +QEMUSizedBuffer *qsb_create(const uint8_t *buffer, size_t len);
> > +QEMUSizedBuffer *qsb_clone(const QEMUSizedBuffer *);
> > +void qsb_free(QEMUSizedBuffer *);
> > +size_t qsb_set_length(QEMUSizedBuffer *qsb, size_t length);
> 
> Just on the prototype, I wonder if this should return ssize_t, to allow
> for the possibility of failure...
> 
> > +/**
> > + * Create a QEMUSizedBuffer
> > + * This type of buffer uses scatter-gather lists internally and
> > + * can grow to any size. Any data array in the scatter-gather list
> > + * can hold different amount of bytes.
> > + *
> > + * @buffer: Optional buffer to copy into the QSB
> > + * @len: size of initial buffer; if @buffer is given, buffer must
> > + *       hold at least len bytes
> > + *
> > + * Returns a pointer to a QEMUSizedBuffer
> > + */
> > +QEMUSizedBuffer *qsb_create(const uint8_t *buffer, size_t len)
> > +{
> > +    QEMUSizedBuffer *qsb;
> > +    size_t alloc_len, num_chunks, i, to_copy;
> > +    size_t chunk_size = (len > QSB_MAX_CHUNK_SIZE)
> > +                        ? QSB_MAX_CHUNK_SIZE
> > +                        : QSB_CHUNK_SIZE;
> > +
> > +    if (len == 0) {
> > +        /* we want to allocate at least one chunk */
> > +        len = QSB_CHUNK_SIZE;
> > +    }
> 
> So if I call qsb_create("", 0), len is now QSB_CHUNK_SIZE...
> 
> > +
> > +    num_chunks = DIV_ROUND_UP(len, chunk_size);
> 
> ...num_chunks is now 1...
> 
> > +    alloc_len = num_chunks * chunk_size;
> > +
> > +    qsb = g_new0(QEMUSizedBuffer, 1);
> > +    qsb->iov = g_new0(struct iovec, num_chunks);
> > +    qsb->n_iov = num_chunks;
> > +
> > +    for (i = 0; i < num_chunks; i++) {
> > +        qsb->iov[i].iov_base = g_malloc0(chunk_size);
> > +        qsb->iov[i].iov_len = chunk_size;
> > +        if (buffer) {
> 
> ...we enter the loop, and have a buffer to copy...
> 
> > +            to_copy = (len - qsb->used) > chunk_size
> > +                      ? chunk_size : (len - qsb->used);
> > +            memcpy(qsb->iov[i].iov_base, &buffer[qsb->used], to_copy);
> 
> ...and proceed to dereference QSB_CHUNK_SIZE bytes beyond the end of the
> empty string that we are converting to a buffer.  Oops.
> 
> Might be as simple as adding if (buffer) { assert(len); } before
> modifying len.
> 
> > +/**
> > + * Set the length of the buffer; the primary usage of this
> > + * function is to truncate the number of used bytes in the buffer.
> > + * The size will not be extended beyond the current number of
> > + * allocated bytes in the QEMUSizedBuffer.
> > + *
> > + * @qsb: A QEMUSizedBuffer
> > + * @new_len: The new length of bytes in the buffer
> > + *
> > + * Returns the number of bytes the buffer was truncated or extended
> > + * to.
> 
> Confusing; I'd suggest:
> 
> Returns the resulting (possibly new) size of the buffer.  Oh, and my
> earlier question appears to be answered - this function can't fail.
> 
> > +ssize_t qsb_get_buffer(const QEMUSizedBuffer *qsb, off_t start,
> > +                       size_t count, uint8_t **buf)
> > +{
> > +    uint8_t *buffer;
> > +    const struct iovec *iov;
> > +    size_t to_copy, all_copy;
> > +    ssize_t index;
> > +    off_t s_off;
> > +    off_t d_off = 0;
> > +    char *s;
> > +
> > +    if (start > qsb->used) {
> > +        return 0;
> > +    }
> 
> It feels confusing that this can return 0 while leaving buf un-malloc'd.
>  It's probably simpler to callers if a non-negative return value ALWAYS
> guarantees that buf is valid.
> 
> 
> > +
> > +    index = qsb_get_iovec(qsb, start, &s_off);
> > +    if (index < 0) {
> > +        return 0;
> > +    }
> 
> Wait - how is a negative value possible?  Since we already checked
> against qsb->used, can't this be assert(index >= 0)?
> 
> > +static ssize_t qsb_grow(QEMUSizedBuffer *qsb, size_t new_size)
> > +{
> > +    size_t needed_chunks, i;
> > +    size_t chunk_size = QSB_CHUNK_SIZE;
> > +
> > +    if (qsb->size < new_size) {
> > +        needed_chunks = DIV_ROUND_UP(new_size - qsb->size,
> > +                                     chunk_size);
> > +
> 
> Hmm. Original creation will use chunks larger than QSB_CHUNK_SIZE (up to
> the maximum chunk size), presumably for efficiency.  Should this
> function likewise use larger chunks if the size to be grown is larger
> than the default chunk size?
> 
> > +        qsb->iov = g_realloc_n(qsb->iov, qsb->n_iov + needed_chunks,
> > +                               sizeof(struct iovec));
> > +        if (qsb->iov == NULL) {
> 
> Wait. Can g_realloc_n fail? I know that g_malloc can't fail, and you
> have to use g_try_malloc instead.  Do you need to try a different variant?
> 
> > +            return -ENOMEM;
> > +        }
> > +
> > +        for (i = qsb->n_iov; i < qsb->n_iov + needed_chunks; i++) {
> > +            qsb->iov[i].iov_base = g_malloc0(chunk_size);
> 
> Wait.  Why do you care about realloc'ing the iov array failing
> (unlikely, as that's a relatively small allocation unlikely to be more
> than a megabyte) but completely ignore the possibility of abort'ing when
> g_malloc0 of a chunk size fails (much bigger, and therefore more likely
> to fail if memory pressure is high)?  Either abort is always fine (no
> need to ever report -ENOMEM) or you should be using a different
> allocation function here.
> 
> > +/**
> > + * Write into the QEMUSizedBuffer at a given position and a given
> > + * number of bytes. This function will automatically grow the
> > + * QEMUSizedBuffer.
> > + *
> > + * @qsb: A QEMUSizedBuffer
> > + * @source: A byte array to copy data from
> > + * @pos: The position withing the @qsb to write data to
> 
> s/withing/within/
> 
> > + * @size: The number of bytes to copy into the @qsb
> > + *
> > + * Returns an error code in case of memory allocation failure,
> > + * @size otherwise.
> > + */
> > +ssize_t qsb_write_at(QEMUSizedBuffer *qsb, const uint8_t *source,
> > +                     off_t pos, size_t count)
> > +{
> > +    ssize_t rc = qsb_grow(qsb, pos + count);
> > +    size_t to_copy;
> > +    size_t all_copy = count;
> > +    const struct iovec *iov;
> > +    ssize_t index;
> > +    char *dest;
> > +    off_t d_off, s_off = 0;
> > +
> > +    if (rc < 0) {
> > +        return rc;
> > +    }
> > +
> > +    if (pos + count > qsb->used) {
> > +        qsb->used = pos + count;
> > +    }
> > +
> > +    index = qsb_get_iovec(qsb, pos, &d_off);
> > +    if (index < 0) {
> 
> Shouldn't this be an assert, because the prior qsb_grow() should have
> guaranteed that we are large enough?
> 
> > +/**
> > + * Create an exact copy of the given QEMUSizedBuffer.
> 
> s/exact // - it's definitely a copy representing the same contents, but
> might have different boundaries for internal iovs, so omitting the word
> will avoid confusion on whether that was intended.  (If you wanted an
> exact copy, then I would manually allocate each iov to the right size,
> and memcpy contents over from the original, instead of using qsb_create
> and qsb_write_at).
> 
> -- 
> Eric Blake   eblake redhat com    +1-919-301-3266
> Libvirt virtualization library http://libvirt.org
> 


--
Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK

[Qemu-devel] [PATCH v2 2/2] Tests: QEMUSizedBuffer/QEMUBuffer

[Dr. David Alan Gilbert (git)]

From: "Dr. David Alan Gilbert" <dgilbert@redhat.com>

Modify some of tests/test-vmstate.c to use the in memory file based
on QEMUSizedBuffer to provide basic testing of QEMUSizedBuffer and
the associated memory backed QEMUFile type.

Only some of the tests are changed so that the fd backed QEMUFile is
still tested.

Signed-off-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
 tests/Makefile       |  2 +-
 tests/test-vmstate.c | 73 ++++++++++++++++++++++++++--------------------------
 2 files changed, 38 insertions(+), 37 deletions(-)

diff --git a/tests/Makefile b/tests/Makefile
index 4b2e1bb..f3d32ba 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -253,7 +253,7 @@ tests/test-qdev-global-props$(EXESUF): tests/test-qdev-global-props.o \
 	libqemuutil.a libqemustub.a
 tests/test-vmstate$(EXESUF): tests/test-vmstate.o \
 	vmstate.o qemu-file.o \
-	libqemuutil.a
+	libqemuutil.a libqemustub.a
 
 tests/test-qapi-types.c tests/test-qapi-types.h :\
 $(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-types.py
diff --git a/tests/test-vmstate.c b/tests/test-vmstate.c
index d72c64c..716d034 100644
--- a/tests/test-vmstate.c
+++ b/tests/test-vmstate.c
@@ -43,6 +43,12 @@ void yield_until_fd_readable(int fd)
     select(fd + 1, &fds, NULL, NULL, NULL);
 }
 
+/*
+ * Some tests use 'open_test_file' to work on a real fd, some use
+ * an in memory file (QEMUSizedBuffer+qemu_bufopen); we could pick one
+ * but this way we test both.
+ */
+
 /* Duplicate temp_fd and seek to the beginning of the file */
 static QEMUFile *open_test_file(bool write)
 {
@@ -54,6 +60,29 @@ static QEMUFile *open_test_file(bool write)
     return qemu_fdopen(fd, write ? "wb" : "rb");
 }
 
+/* Open a read-only qemu-file from an existing memory block */
+static QEMUFile *open_mem_file_read(const void *data, size_t len)
+{
+    /* The qsb gets freed by qemu_fclose */
+    QEMUSizedBuffer *qsb = qsb_create(data, len);
+
+    return qemu_bufopen("r", qsb);
+}
+
+/*
+ * Check that the contents of the memory-buffered file f match
+ * the given size/data.
+ */
+static void check_mem_file(QEMUFile *f, void *data, size_t size)
+{
+    uint8_t *result = NULL; /* qsb_get_buffer allocs a buffer */
+    const QEMUSizedBuffer *qsb = qemu_buf_get(f);
+    g_assert_cmpint(qsb_get_length(qsb), ==, size);
+    g_assert_cmpint(qsb_get_buffer(qsb, 0, size, &result), ==, size);
+    g_assert_cmpint(memcmp(result, data, size), ==, 0);
+    g_free(result);
+}
+
 #define SUCCESS(val) \
     g_assert_cmpint((val), ==, 0)
 
@@ -371,14 +400,12 @@ static const VMStateDescription vmstate_skipping = {
 
 static void test_save_noskip(void)
 {
-    QEMUFile *fsave = open_test_file(true);
+    QEMUFile *fsave = qemu_bufopen("w", NULL);
     TestStruct obj = { .a = 1, .b = 2, .c = 3, .d = 4, .e = 5, .f = 6,
                        .skip_c_e = false };
     vmstate_save_state(fsave, &vmstate_skipping, &obj);
     g_assert(!qemu_file_get_error(fsave));
-    qemu_fclose(fsave);
 
-    QEMUFile *loading = open_test_file(false);
     uint8_t expected[] = {
         0, 0, 0, 1,             /* a */
         0, 0, 0, 2,             /* b */
@@ -387,52 +414,31 @@ static void test_save_noskip(void)
         0, 0, 0, 5,             /* e */
         0, 0, 0, 0, 0, 0, 0, 6, /* f */
     };
-    uint8_t result[sizeof(expected)];
-    g_assert_cmpint(qemu_get_buffer(loading, result, sizeof(result)), ==,
-                    sizeof(result));
-    g_assert(!qemu_file_get_error(loading));
-    g_assert_cmpint(memcmp(result, expected, sizeof(result)), ==, 0);
-
-    /* Must reach EOF */
-    qemu_get_byte(loading);
-    g_assert_cmpint(qemu_file_get_error(loading), ==, -EIO);
-
-    qemu_fclose(loading);
+    check_mem_file(fsave, expected, sizeof(expected));
+    qemu_fclose(fsave);
 }
 
 static void test_save_skip(void)
 {
-    QEMUFile *fsave = open_test_file(true);
+    QEMUFile *fsave = qemu_bufopen("w", NULL);
     TestStruct obj = { .a = 1, .b = 2, .c = 3, .d = 4, .e = 5, .f = 6,
                        .skip_c_e = true };
     vmstate_save_state(fsave, &vmstate_skipping, &obj);
     g_assert(!qemu_file_get_error(fsave));
-    qemu_fclose(fsave);
 
-    QEMUFile *loading = open_test_file(false);
     uint8_t expected[] = {
         0, 0, 0, 1,             /* a */
         0, 0, 0, 2,             /* b */
         0, 0, 0, 0, 0, 0, 0, 4, /* d */
         0, 0, 0, 0, 0, 0, 0, 6, /* f */
     };
-    uint8_t result[sizeof(expected)];
-    g_assert_cmpint(qemu_get_buffer(loading, result, sizeof(result)), ==,
-                    sizeof(result));
-    g_assert(!qemu_file_get_error(loading));
-    g_assert_cmpint(memcmp(result, expected, sizeof(result)), ==, 0);
-
-
-    /* Must reach EOF */
-    qemu_get_byte(loading);
-    g_assert_cmpint(qemu_file_get_error(loading), ==, -EIO);
+    check_mem_file(fsave, expected, sizeof(expected));
 
-    qemu_fclose(loading);
+    qemu_fclose(fsave);
 }
 
 static void test_load_noskip(void)
 {
-    QEMUFile *fsave = open_test_file(true);
     uint8_t buf[] = {
         0, 0, 0, 10,             /* a */
         0, 0, 0, 20,             /* b */
@@ -442,10 +448,8 @@ static void test_load_noskip(void)
         0, 0, 0, 0, 0, 0, 0, 60, /* f */
         QEMU_VM_EOF, /* just to ensure we won't get EOF reported prematurely */
     };
-    qemu_put_buffer(fsave, buf, sizeof(buf));
-    qemu_fclose(fsave);
 
-    QEMUFile *loading = open_test_file(false);
+    QEMUFile *loading = open_mem_file_read(buf, sizeof(buf));
     TestStruct obj = { .skip_c_e = false };
     vmstate_load_state(loading, &vmstate_skipping, &obj, 2);
     g_assert(!qemu_file_get_error(loading));
@@ -460,7 +464,6 @@ static void test_load_noskip(void)
 
 static void test_load_skip(void)
 {
-    QEMUFile *fsave = open_test_file(true);
     uint8_t buf[] = {
         0, 0, 0, 10,             /* a */
         0, 0, 0, 20,             /* b */
@@ -468,10 +471,8 @@ static void test_load_skip(void)
         0, 0, 0, 0, 0, 0, 0, 60, /* f */
         QEMU_VM_EOF, /* just to ensure we won't get EOF reported prematurely */
     };
-    qemu_put_buffer(fsave, buf, sizeof(buf));
-    qemu_fclose(fsave);
 
-    QEMUFile *loading = open_test_file(false);
+    QEMUFile *loading = open_mem_file_read(buf, sizeof(buf));
     TestStruct obj = { .skip_c_e = true, .c = 300, .e = 500 };
     vmstate_load_state(loading, &vmstate_skipping, &obj, 2);
     g_assert(!qemu_file_get_error(loading));
-- 
1.9.3