Re: [Qemu-devel] [PATCH 2/2] migration-test: Add a test for fd protocol

[Yury Kotov <yury-kotov@yandex-team.ru>]

27.05.2019, 12:35, "Yury Kotov" <yury-kotov@yandex-team.ru>:
> Signed-off-by: Yury Kotov <yury-kotov@yandex-team.ru>
> ---
>  tests/libqtest.c | 83 ++++++++++++++++++++++++++++++--
>  tests/libqtest.h | 51 +++++++++++++++++++-
>  tests/migration-test.c | 107 +++++++++++++++++++++++++++++++++++++++--
>  3 files changed, 233 insertions(+), 8 deletions(-)
>
> diff --git a/tests/libqtest.c b/tests/libqtest.c
> index 8ac0c02af4..de8468d213 100644
> --- a/tests/libqtest.c
> +++ b/tests/libqtest.c
> @@ -32,6 +32,7 @@
>
>  #define MAX_IRQ 256
>  #define SOCKET_TIMEOUT 50
> +#define SOCKET_MAX_FDS 16
>
>  QTestState *global_qtest;
>
> @@ -391,6 +392,43 @@ static void GCC_FMT_ATTR(2, 3) qtest_sendf(QTestState *s, const char *fmt, ...)
>      va_end(ap);
>  }
>
> +static void socket_send_fds(int fd, int *fds, size_t fds_num,
> + const char *buf, size_t buf_size)
> +{
> +#ifndef WIN32
> + ssize_t ret;
> + struct msghdr msg = { 0 };
> + char control[CMSG_SPACE(sizeof(int) * SOCKET_MAX_FDS)] = { 0 };
> + size_t fdsize = sizeof(int) * fds_num;
> + struct cmsghdr *cmsg;
> + struct iovec iov = { .iov_base = (char *)buf, .iov_len = buf_size };
> +
> + msg.msg_iov = &iov;
> + msg.msg_iovlen = 1;
> +
> + if (fds && fds_num > 0) {
> + g_assert_cmpuint(fds_num, <, SOCKET_MAX_FDS);
> +
> + msg.msg_control = control;
> + msg.msg_controllen = CMSG_SPACE(fdsize);
> +
> + cmsg = CMSG_FIRSTHDR(&msg);
> + cmsg->cmsg_len = CMSG_LEN(fdsize);
> + cmsg->cmsg_level = SOL_SOCKET;
> + cmsg->cmsg_type = SCM_RIGHTS;
> + memcpy(CMSG_DATA(cmsg), fds, fdsize);
> + }
> +
> + do {
> + ret = sendmsg(fd, &msg, 0);
> + } while (ret < 0 && errno == EINTR);
> + g_assert_cmpint(ret, >, 0);
> +#else
> + g_test_skip("sendmsg is not supported under Win32");
> + return;
> +#endif
> +}
> +
>  static GString *qtest_recv_line(QTestState *s)
>  {
>      GString *line;
> @@ -545,7 +583,8 @@ QDict *qtest_qmp_receive(QTestState *s)
>   * in the case that they choose to discard all replies up until
>   * a particular EVENT is received.
>   */
> -void qmp_fd_vsend(int fd, const char *fmt, va_list ap)
> +void qmp_fd_vsend_fds(int fd, int *fds, size_t fds_num,
> + const char *fmt, va_list ap)
>  {
>      QObject *qobj;
>
> @@ -569,25 +608,49 @@ void qmp_fd_vsend(int fd, const char *fmt, va_list ap)
>              fprintf(stderr, "%s", str);
>          }
>          /* Send QMP request */
> - socket_send(fd, str, qstring_get_length(qstr));
> + if (fds && fds_num > 0) {
> + socket_send_fds(fd, fds, fds_num, str, qstring_get_length(qstr));
> + } else {
> + socket_send(fd, str, qstring_get_length(qstr));
> + }
>
>          qobject_unref(qstr);
>          qobject_unref(qobj);
>      }
>  }
>
> +void qmp_fd_vsend(int fd, const char *fmt, va_list ap)
> +{
> + qmp_fd_vsend_fds(fd, NULL, 0, fmt, ap);
> +}
> +
> +void qtest_qmp_vsend_fds(QTestState *s, int *fds, size_t fds_num,
> + const char *fmt, va_list ap)
> +{
> + qmp_fd_vsend_fds(s->qmp_fd, fds, fds_num, fmt, ap);
> +}
> +
>  void qtest_qmp_vsend(QTestState *s, const char *fmt, va_list ap)
>  {
> - qmp_fd_vsend(s->qmp_fd, fmt, ap);
> + qmp_fd_vsend_fds(s->qmp_fd, NULL, 0, fmt, ap);
>  }
>
>  QDict *qmp_fdv(int fd, const char *fmt, va_list ap)
>  {
> - qmp_fd_vsend(fd, fmt, ap);
> + qmp_fd_vsend_fds(fd, NULL, 0, fmt, ap);
>
>      return qmp_fd_receive(fd);
>  }
>
> +QDict *qtest_vqmp_fds(QTestState *s, int *fds, size_t fds_num,
> + const char *fmt, va_list ap)
> +{
> + qtest_qmp_vsend_fds(s, fds, fds_num, fmt, ap);
> +
> + /* Receive reply */
> + return qtest_qmp_receive(s);
> +}
> +
>  QDict *qtest_vqmp(QTestState *s, const char *fmt, va_list ap)
>  {
>      qtest_qmp_vsend(s, fmt, ap);
> @@ -616,6 +679,18 @@ void qmp_fd_send(int fd, const char *fmt, ...)
>      va_end(ap);
>  }
>
> +QDict *qtest_qmp_fds(QTestState *s, int *fds, size_t fds_num,
> + const char *fmt, ...)
> +{
> + va_list ap;
> + QDict *response;
> +
> + va_start(ap, fmt);
> + response = qtest_vqmp_fds(s, fds, fds_num, fmt, ap);
> + va_end(ap);
> + return response;
> +}
> +
>  QDict *qtest_qmp(QTestState *s, const char *fmt, ...)
>  {
>      va_list ap;
> diff --git a/tests/libqtest.h b/tests/libqtest.h
> index a98ea15b7d..e61ebaced1 100644
> --- a/tests/libqtest.h
> +++ b/tests/libqtest.h
> @@ -84,6 +84,21 @@ QTestState *qtest_init_with_serial(const char *extra_args, int *sock_fd);
>   */
>  void qtest_quit(QTestState *s);
>
> +/**
> + * qtest_qmp_fds:
> + * @s: #QTestState instance to operate on.
> + * @fds: array of file descriptors
> + * @fds_num: number of elements in @fds
> + * @fmt...: QMP message to send to qemu, formatted like
> + * qobject_from_jsonf_nofail(). See parse_escape() for what's
> + * supported after '%'.
> + *
> + * Sends a QMP message to QEMU with fds and returns the response.
> + */
> +QDict *qtest_qmp_fds(QTestState *s, int *fds, size_t fds_num,
> + const char *fmt, ...)
> + GCC_FMT_ATTR(4, 5);
> +
>  /**
>   * qtest_qmp:
>   * @s: #QTestState instance to operate on.
> @@ -120,7 +135,23 @@ void qtest_qmp_send_raw(QTestState *s, const char *fmt, ...)
>      GCC_FMT_ATTR(2, 3);
>
>  /**
> - * qtest_qmpv:
> + * qtest_vqmp_fds:
> + * @s: #QTestState instance to operate on.
> + * @fds: array of file descriptors
> + * @fds_num: number of elements in @fds
> + * @fmt: QMP message to send to QEMU, formatted like
> + * qobject_from_jsonf_nofail(). See parse_escape() for what's
> + * supported after '%'.
> + * @ap: QMP message arguments
> + *
> + * Sends a QMP message to QEMU with fds and returns the response.
> + */
> +QDict *qtest_vqmp_fds(QTestState *s, int *fds, size_t fds_num,
> + const char *fmt, va_list ap)
> + GCC_FMT_ATTR(4, 0);
> +
> +/**
> + * qtest_vqmp:
>   * @s: #QTestState instance to operate on.
>   * @fmt: QMP message to send to QEMU, formatted like
>   * qobject_from_jsonf_nofail(). See parse_escape() for what's
> @@ -132,6 +163,22 @@ void qtest_qmp_send_raw(QTestState *s, const char *fmt, ...)
>  QDict *qtest_vqmp(QTestState *s, const char *fmt, va_list ap)
>      GCC_FMT_ATTR(2, 0);
>
> +/**
> + * qtest_qmp_vsend_fds:
> + * @s: #QTestState instance to operate on.
> + * @fds: array of file descriptors
> + * @fds_num: number of elements in @fds
> + * @fmt: QMP message to send to QEMU, formatted like
> + * qobject_from_jsonf_nofail(). See parse_escape() for what's
> + * supported after '%'.
> + * @ap: QMP message arguments
> + *
> + * Sends a QMP message to QEMU and leaves the response in the stream.
> + */
> +void qtest_qmp_vsend_fds(QTestState *s, int *fds, size_t fds_num,
> + const char *fmt, va_list ap)
> + GCC_FMT_ATTR(4, 0);
> +
>  /**
>   * qtest_qmp_vsend:
>   * @s: #QTestState instance to operate on.
> @@ -877,6 +924,8 @@ static inline int64_t clock_step(int64_t step)
>  }
>
>  QDict *qmp_fd_receive(int fd);
> +void qmp_fd_vsend_fds(int fd, int *fds, size_t fds_num,
> + const char *fmt, va_list ap) GCC_FMT_ATTR(4, 0);
>  void qmp_fd_vsend(int fd, const char *fmt, va_list ap) GCC_FMT_ATTR(2, 0);
>  void qmp_fd_send(int fd, const char *fmt, ...) GCC_FMT_ATTR(2, 3);
>  void qmp_fd_send_raw(int fd, const char *fmt, ...) GCC_FMT_ATTR(2, 3);
> diff --git a/tests/migration-test.c b/tests/migration-test.c
> index bd3f5c3125..b0136eb54f 100644
> --- a/tests/migration-test.c
> +++ b/tests/migration-test.c
> @@ -174,6 +174,21 @@ static void stop_cb(void *opaque, const char *name, QDict *data)
>      }
>  }
>
> +/*
> + * Events can get in the way of responses we are actually waiting for.
> + */
> +GCC_FMT_ATTR(3, 4)
> +static QDict *wait_command_fd(QTestState *who, int fd, const char *command, ...)
> +{
> + va_list ap;
> +
> + va_start(ap, command);
> + qtest_qmp_vsend_fds(who, &fd, 1, command, ap);
> + va_end(ap);
> +
> + return qtest_qmp_receive_success(who, stop_cb, NULL);
> +}
> +
>  /*
>   * Events can get in the way of responses we are actually waiting for.
>   */
> @@ -474,12 +489,12 @@ static void migrate(QTestState *who, const char *uri, const char *fmt, ...)
>
>      va_start(ap, fmt);
>      args = qdict_from_vjsonf_nofail(fmt, ap);
> - va_end(ap);
> -
>      g_assert(!qdict_haskey(args, "uri"));
>      qdict_put_str(args, "uri", uri);
> + va_end(ap);
> +
> + rsp = qtest_qmp(who, "{ 'execute': 'migrate', 'arguments': %p}", args);
>
> - rsp = qmp("{ 'execute': 'migrate', 'arguments': %p}", args);

These changes in migrate are not needed. I forgot to revert them before sending
the series... I'll revert them in v2.

>      g_assert(qdict_haskey(rsp, "return"));
>      qobject_unref(rsp);
>  }
> @@ -1027,6 +1042,91 @@ static void test_precopy_tcp(void)
>      g_free(uri);
>  }
>
> +static void test_migrate_fd_proto(void)
> +{
> + QTestState *from, *to;
> + int ret;
> + int pair[2];
> + QDict *rsp;
> + const char *error_desc;
> +
> + if (test_migrate_start(&from, &to, "defer", false, false)) {
> + return;
> + }
> +
> + /*
> + * We want to pick a speed slow enough that the test completes
> + * quickly, but that it doesn't complete precopy even on a slow
> + * machine, so also set the downtime.
> + */
> + /* 1 ms should make it not converge */
> + migrate_set_parameter(from, "downtime-limit", 1);
> + /* 1GB/s */
> + migrate_set_parameter(from, "max-bandwidth", 1000000000);
> +
> + /* Wait for the first serial output from the source */
> + wait_for_serial("src_serial");
> +
> + /* Create two connected sockets for migration */
> + ret = socketpair(PF_LOCAL, SOCK_STREAM, 0, pair);
> + g_assert_cmpint(ret, ==, 0);
> +
> + /* Send the 1st socket to the target */
> + rsp = wait_command_fd(to, pair[0],
> + "{ 'execute': 'getfd',"
> + " 'arguments': { 'fdname': 'fd-mig' }}");
> + qobject_unref(rsp);
> + close(pair[0]);
> +
> + /* Start incoming migration from the 1st socket */
> + rsp = wait_command(to, "{ 'execute': 'migrate-incoming',"
> + " 'arguments': { 'uri': 'fd:fd-mig' }}");
> + qobject_unref(rsp);
> +
> + /* Send the 2nd socket to the target */
> + rsp = wait_command_fd(from, pair[1],
> + "{ 'execute': 'getfd',"
> + " 'arguments': { 'fdname': 'fd-mig' }}");
> + qobject_unref(rsp);
> + close(pair[1]);
> +
> + /* Start migration to the 2nd socket*/
> + migrate(from, "fd:fd-mig", "{}");
> +
> + wait_for_migration_pass(from);
> +
> + /* 300ms should converge */
> + migrate_set_parameter(from, "downtime-limit", 300);
> +
> + if (!got_stop) {
> + qtest_qmp_eventwait(from, "STOP");
> + }
> + qtest_qmp_eventwait(to, "RESUME");
> +
> + /* Test closing fds */
> + /* We assume, that QEMU removes named fd from its list,
> + * so this should fail */
> + rsp = qtest_qmp(from, "{ 'execute': 'closefd',"
> + " 'arguments': { 'fdname': 'fd-mig' }}");
> + g_assert_true(qdict_haskey(rsp, "error"));
> + error_desc = qdict_get_str(qdict_get_qdict(rsp, "error"), "desc");
> + g_assert_cmpstr(error_desc, ==, "File descriptor named 'fd-mig' not found");
> + qobject_unref(rsp);
> +
> + rsp = qtest_qmp(to, "{ 'execute': 'closefd',"
> + " 'arguments': { 'fdname': 'fd-mig' }}");
> + g_assert_true(qdict_haskey(rsp, "error"));
> + error_desc = qdict_get_str(qdict_get_qdict(rsp, "error"), "desc");
> + g_assert_cmpstr(error_desc, ==, "File descriptor named 'fd-mig' not found");
> + qobject_unref(rsp);
> +
> +
> + /* Complete migration */
> + wait_for_serial("dest_serial");
> + wait_for_migration_complete(from);
> + test_migrate_end(from, to, true);
> +}
> +
>  int main(int argc, char **argv)
>  {
>      char template[] = "/tmp/migration-test-XXXXXX";
> @@ -1081,6 +1181,7 @@ int main(int argc, char **argv)
>      qtest_add_func("/migration/precopy/tcp", test_precopy_tcp);
>      /* qtest_add_func("/migration/ignore_shared", test_ignore_shared); */
>      qtest_add_func("/migration/xbzrle/unix", test_xbzrle_unix);
> + qtest_add_func("/migration/fd_proto", test_migrate_fd_proto);
>
>      ret = g_test_run();
>
> --
> 2.21.0

Regards,
Yury