[Qemu-devel] [PATCH RFC] tcmu: Introduce qemu-tcmu

* [Qemu-devel] [PATCH RFC] tcmu: Introduce qemu-tcmu
@ 2016-10-19 10:08 Fam Zheng
  2016-10-19 10:38 ` no-reply
                   ` (2 more replies)
  0 siblings, 3 replies; 9+ messages in thread
From: Fam Zheng @ 2016-10-19 10:08 UTC (permalink / raw)
  To: qemu-devel
  Cc: Paolo Bonzini, Pranith Kumar Karampuri, Fam Zheng, Vijay Bellur,
	Huamin Chen, Andy Grover, Stefan Hajnoczi

libtcmu is a Linux library for userspace programs to handle TCMU
protocol, which is a SCSI transport between the target_core_user.ko and
a userspace backend implementation. The former can be seen as a kernel
SCSI Low-Level-Driver that forwards scsi requests to userspace via a
UIO ring buffer. By linking to libtcmu, a program can serve as a scsi
HBA.

This patch adds qemu-tcmu utility that serves as a LIO userspace
backend.  Apart from how it interacts with the rest of the world, the
role of qemu-tcmu is much alike to qemu-nbd: it can export any
format/protocol that QEMU supports (in iscsi/loopback instead of NBD),
for local or remote access. The main advantage with qemu-tcmu lies in
the more flexible command protocol (SCSI) and more flexible iscsi or
loopback frontends, the latter of which can theoretically allow zero-copy
I/O from local machine, which makes qemu-tcmu potentially possible to
serve data in better performance.

RFC because only minimal scsi commands are now handled. The plan is to
refactor and reuse scsi-disk emulation code. Also there is no test code.

Similar to NBD, there will be QMP commands to start built-in targets so
that guest images can be exported as well.

Usage example script (tested on Fedora 24):

    set -e

    # For targetcli integration
    sudo systemctl start tcmu-runner

    qemu-img create test-img 1G

    # libtcmu needs to open UIO, give it access
    sudo chmod 777 /dev/uio0

    qemu-tcmu test-img &

    sleep 1

    IQN=iqn.2003-01.org.linux-iscsi.lemon.x8664:sn.4939fc29108f

    # Check that we have initialized correctly
    sudo targetcli ls | grep -q user:qemu

    # Create iscsi target
    sudo targetcli ls | grep -q $IQN || sudo targetcli /iscsi create wwn=$IQN
    sudo targetcli /iscsi/$IQN/tpg1 set attribute \
        authentication=0 generate_node_acls=1 demo_mode_write_protect=0 \
        prod_mode_write_protect=0

    # Create the qemu-tcmu target
    sudo targetcli ls | grep -q d0 || \
        sudo targetcli /backstores/user:qemu create d0 1G @drive

    # Export it as an iscsi LUN
    sudo targetcli ls | grep -q lun0 || \
        sudo targetcli /iscsi/$IQN/tpg1/luns create storage_object=/backstores/user:qemu/d0

    # Inspect the nodes again
    sudo targetcli ls

    # Test that the LIO export works
    qemu-img info iscsi://127.0.0.1/$IQN/0
    qemu-io iscsi://127.0.0.1/$IQN/0 \
        -c 'read -P 1 4k 1k' \
        -c 'write -P 2 4k 1k' \
        -c 'read -P 2 4k 1k' \

Signed-off-by: Fam Zheng <famz@redhat.com>
---
 Makefile            |   1 +
 Makefile.objs       |   4 +-
 configure           |  45 ++++++
 include/scsi/tcmu.h |  11 ++
 qemu-tcmu.c         | 401 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 scsi/Makefile.objs  |   5 +
 scsi/tcmu.c         | 337 +++++++++++++++++++++++++++++++++++++++++++
 7 files changed, 802 insertions(+), 2 deletions(-)
 create mode 100644 include/scsi/tcmu.h
 create mode 100644 qemu-tcmu.c
 create mode 100644 scsi/Makefile.objs
 create mode 100644 scsi/tcmu.c

diff --git a/Makefile b/Makefile
index 3bcb056..9b7812e 100644
--- a/Makefile
+++ b/Makefile
@@ -251,6 +251,7 @@ qemu-img.o: qemu-img-cmds.h
 qemu-img$(EXESUF): qemu-img.o $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) libqemuutil.a libqemustub.a
 qemu-nbd$(EXESUF): qemu-nbd.o $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) libqemuutil.a libqemustub.a
 qemu-io$(EXESUF): qemu-io.o $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) libqemuutil.a libqemustub.a
+qemu-tcmu$(EXESUF): qemu-tcmu.o $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) libqemuutil.a libqemustub.a
 
 qemu-bridge-helper$(EXESUF): qemu-bridge-helper.o libqemuutil.a libqemustub.a
 
diff --git a/Makefile.objs b/Makefile.objs
index 69fdd48..c046afe 100644
--- a/Makefile.objs
+++ b/Makefile.objs
@@ -13,11 +13,11 @@ block-obj-y += block.o blockjob.o
 block-obj-y += main-loop.o iohandler.o qemu-timer.o
 block-obj-$(CONFIG_POSIX) += aio-posix.o
 block-obj-$(CONFIG_WIN32) += aio-win32.o
-block-obj-y += block/
+block-obj-y += block/ scsi/
 block-obj-y += qemu-io-cmds.o
 block-obj-$(CONFIG_REPLICATION) += replication.o
 
-block-obj-m = block/
+block-obj-m = block/ scsi/
 
 #######################################################################
 # crypto-obj-y is code used by both qemu system emulation and qemu-img
diff --git a/configure b/configure
index dd9e679..04b4951 100755
--- a/configure
+++ b/configure
@@ -209,6 +209,7 @@ netmap="no"
 pixman=""
 sdl=""
 sdlabi=""
+tcmu=""
 virtfs=""
 vnc="yes"
 sparse="no"
@@ -829,6 +830,10 @@ for opt do
   ;;
   --without-pixman) pixman="none"
   ;;
+  --enable-tcmu) tcmu="yes"
+  ;;
+  --disable-tcmu) tcmu="no"
+  ;;
   --disable-sdl) sdl="no"
   ;;
   --enable-sdl) sdl="yes"
@@ -3108,6 +3113,36 @@ else
 fi
 
 ##########################################
+# tcmu support probe
+
+if test "$tcmu" != "no"; then
+  # Sanity check for gio-unix-2.0 (part of glib2), cannot fail unless something
+  # is very wrong.
+  if ! $pkg_config gio-unix-2.0; then
+    error_exit "glib is required to compile QEMU"
+  fi
+  cat > $TMPC <<EOF
+#include <stdio.h>
+#include <libtcmu.h>
+
+int main(int argc, char **argv)
+{
+  struct tcmulib_context *ctx = tcmulib_initialize(NULL, 0, NULL);
+  tcmulib_register(ctx);
+  return ctx != NULL;
+}
+EOF
+  if compile_prog "" "-ltcmu" ; then
+    tcmu=yes
+    tcmu_libs="-ltcmu"
+  elif test "$tcmu" == "yes"; then
+    feature_not_found "libtcmu" "Install libtcmu devel (>=1.0.5)"
+  else
+    tcmu=no
+  fi
+fi
+
+##########################################
 # libcap probe
 
 if test "$cap" != "no" ; then
@@ -4660,6 +4695,9 @@ if test "$want_tools" = "yes" ; then
     tools="qemu-nbd\$(EXESUF) $tools"
     tools="ivshmem-client\$(EXESUF) ivshmem-server\$(EXESUF) $tools"
   fi
+  if [ "$linux" = "yes" -a "$tcmu" = "yes" ] ; then
+    tools="qemu-tcmu\$(EXESUF) $tools"
+  fi
 fi
 if test "$softmmu" = yes ; then
   if test "$virtfs" != no ; then
@@ -4960,6 +4998,7 @@ echo "tcmalloc support  $tcmalloc"
 echo "jemalloc support  $jemalloc"
 echo "avx2 optimization $avx2_opt"
 echo "replication support $replication"
+echo "tcmu support      $tcmu"
 
 if test "$sdl_too_old" = "yes"; then
 echo "-> Your SDL version is too old - please upgrade to have SDL support"
@@ -5480,6 +5519,12 @@ if test "$libssh2" = "yes" ; then
   echo "LIBSSH2_LIBS=$libssh2_libs" >> $config_host_mak
 fi
 
+if test "$tcmu" = "yes" ; then
+  echo "CONFIG_TCMU=m" >> $config_host_mak
+  echo "TCMU_CFLAGS=$tcmu_cflags" >> $config_host_mak
+  echo "TCMU_LIBS=$tcmu_libs" >> $config_host_mak
+fi
+
 # USB host support
 if test "$libusb" = "yes"; then
   echo "HOST_USB=libusb legacy" >> $config_host_mak
diff --git a/include/scsi/tcmu.h b/include/scsi/tcmu.h
new file mode 100644
index 0000000..dde3435
--- /dev/null
+++ b/include/scsi/tcmu.h
@@ -0,0 +1,11 @@
+#ifndef QEMU_TCMU_H
+#define QEMU_TCMU_H
+
+#include "qemu-common.h"
+
+typedef struct TCMUExport TCMUExport;
+
+void qemu_tcmu_start(const char *subtype, Error **errp);
+TCMUExport *qemu_tcmu_export(BlockBackend *blk, bool writable, Error **errp);
+
+#endif
diff --git a/qemu-tcmu.c b/qemu-tcmu.c
new file mode 100644
index 0000000..f69e05a
--- /dev/null
+++ b/qemu-tcmu.c
@@ -0,0 +1,401 @@
+/*
+ *  Copyright 2016  Red Hat, Inc.
+ *
+ *  TCMU Handler Program
+ *
+ *  Authors:
+ *    Fam Zheng <famz@redhat.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; under version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "qemu/cutils.h"
+#include "sysemu/block-backend.h"
+#include "block/block_int.h"
+#include "qemu/main-loop.h"
+#include "qemu/error-report.h"
+#include "qemu/config-file.h"
+#include "qemu/bswap.h"
+#include "qemu/log.h"
+#include "block/snapshot.h"
+#include "qapi/util.h"
+#include "qapi/qmp/qstring.h"
+#include "qom/object_interfaces.h"
+#include "crypto/init.h"
+#include "trace/control.h"
+#include "scsi/tcmu.h"
+#include <getopt.h>
+#include "qemu-version.h"
+
+#define QEMU_TCMU_OPT_CACHE         256
+#define QEMU_TCMU_OPT_AIO           257
+#define QEMU_TCMU_OPT_DISCARD       258
+#define QEMU_TCMU_OPT_DETECT_ZEROES 259
+#define QEMU_TCMU_OPT_OBJECT        260
+#define QEMU_TCMU_OPT_IMAGE_OPTS    261
+
+static TCMUExport *exp;
+static int verbose;
+static char *srcpath;
+
+static void usage(const char *name)
+{
+    (printf) (
+"Usage: %s [OPTIONS] FILE\n"
+"QEMU TCMU Handler\n"
+"\n"
+"  -h, --help                display this help and exit\n"
+"  -V, --version             output version information and exit\n"
+"\n"
+"General purpose options:\n"
+"  -v, --verbose             display extra debugging information\n"
+"  -x, --handler-name=NAME   handler name to be used as the subtype for TCMU\n"
+"  --object type,id=ID,...   define an object such as 'secret' for providing\n"
+"                            passwords and/or encryption keys\n"
+"  -T, --trace [[enable=]<pattern>][,events=<file>][,file=<file>]\n"
+"                            specify tracing options\n"
+"\n"
+"Block device options:\n"
+"  -f, --format=FORMAT       set image format (raw, qcow2, ...)\n"
+"  -r, --read-only           export read-only\n"
+"  -s, --snapshot            use FILE as an external snapshot, create a temporary\n"
+"                            file with backing_file=FILE, redirect the write to\n"
+"                            the temporary one\n"
+"  -l, --load-snapshot=SNAPSHOT_PARAM\n"
+"                            load an internal snapshot inside FILE and export it\n"
+"                            as an read-only device, SNAPSHOT_PARAM format is\n"
+"                            'snapshot.id=[ID],snapshot.name=[NAME]', or\n"
+"                            '[ID_OR_NAME]'\n"
+"  -n, --nocache             disable host cache\n"
+"      --cache=MODE          set cache mode (none, writeback, ...)\n"
+"      --aio=MODE            set AIO mode (native or threads)\n"
+"      --discard=MODE        set discard mode (ignore, unmap)\n"
+"      --detect-zeroes=MODE  set detect-zeroes mode (off, on, unmap)\n"
+"      --image-opts          treat FILE as a full set of image options\n"
+"\n"
+"Report bugs to <qemu-devel@nongnu.org>\n"
+    , name);
+}
+
+static void version(const char *name)
+{
+    printf("%s v" QEMU_VERSION QEMU_PKGVERSION "\n", name);
+}
+
+static enum { RUNNING, TERMINATE, TERMINATING, TERMINATED } state;
+
+static QemuOptsList file_opts = {
+    .name = "file",
+    .implied_opt_name = "file",
+    .head = QTAILQ_HEAD_INITIALIZER(file_opts.head),
+    .desc = {
+        /* no elements => accept any params */
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_object_opts = {
+    .name = "object",
+    .implied_opt_name = "qom-type",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_object_opts.head),
+    .desc = {
+        { }
+    },
+};
+
+int main(int argc, char **argv)
+{
+    BlockBackend *blk;
+    BlockDriverState *bs;
+    QemuOpts *sn_opts = NULL;
+    const char *sn_id_or_name = NULL;
+    const char *sopt = "hVb:o:p:rsnP:c:dvk:e:f:tl:x:T:";
+    bool starting = true;
+    struct option lopt[] = {
+        { "help", no_argument, NULL, 'h' },
+        { "version", no_argument, NULL, 'V' },
+        { "read-only", no_argument, NULL, 'r' },
+        { "snapshot", no_argument, NULL, 's' },
+        { "load-snapshot", required_argument, NULL, 'l' },
+        { "nocache", no_argument, NULL, 'n' },
+        { "cache", required_argument, NULL, QEMU_TCMU_OPT_CACHE },
+        { "aio", required_argument, NULL, QEMU_TCMU_OPT_AIO },
+        { "discard", required_argument, NULL, QEMU_TCMU_OPT_DISCARD },
+        { "detect-zeroes", required_argument, NULL,
+          QEMU_TCMU_OPT_DETECT_ZEROES },
+        { "shared", required_argument, NULL, 'e' },
+        { "format", required_argument, NULL, 'f' },
+        { "verbose", no_argument, NULL, 'v' },
+        { "object", required_argument, NULL, QEMU_TCMU_OPT_OBJECT },
+        { "handler-name", required_argument, NULL, 'x' },
+        { "image-opts", no_argument, NULL, QEMU_TCMU_OPT_IMAGE_OPTS },
+        { "trace", required_argument, NULL, 'T' },
+        { NULL, 0, NULL, 0 }
+    };
+    int ch;
+    int opt_ind = 0;
+    int flags = BDRV_O_RDWR;
+    int ret = 0;
+    bool seen_cache = false;
+    bool seen_discard = false;
+    bool seen_aio = false;
+    const char *fmt = NULL;
+    Error *local_err = NULL;
+    BlockdevDetectZeroesOptions detect_zeroes = BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF;
+    QDict *options = NULL;
+    bool imageOpts = false;
+    bool writethrough = true;
+    char *trace_file = NULL;
+    const char *subtype = "qemu";
+
+    module_call_init(MODULE_INIT_TRACE);
+    qcrypto_init(&error_fatal);
+
+    module_call_init(MODULE_INIT_QOM);
+    qemu_add_opts(&qemu_object_opts);
+    qemu_add_opts(&qemu_trace_opts);
+    qemu_init_exec_dir(argv[0]);
+
+    while ((ch = getopt_long(argc, argv, sopt, lopt, &opt_ind)) != -1) {
+        switch (ch) {
+        case 's':
+            flags |= BDRV_O_SNAPSHOT;
+            break;
+        case 'n':
+            optarg = (char *) "none";
+            /* fallthrough */
+        case QEMU_TCMU_OPT_CACHE:
+            if (seen_cache) {
+                error_report("-n and --cache can only be specified once");
+                exit(EXIT_FAILURE);
+            }
+            seen_cache = true;
+            if (bdrv_parse_cache_mode(optarg, &flags, &writethrough) == -1) {
+                error_report("Invalid cache mode `%s'", optarg);
+                exit(EXIT_FAILURE);
+            }
+            break;
+        case QEMU_TCMU_OPT_AIO:
+            if (seen_aio) {
+                error_report("--aio can only be specified once");
+                exit(EXIT_FAILURE);
+            }
+            seen_aio = true;
+            if (!strcmp(optarg, "native")) {
+                flags |= BDRV_O_NATIVE_AIO;
+            } else if (!strcmp(optarg, "threads")) {
+                /* this is the default */
+            } else {
+               error_report("invalid aio mode `%s'", optarg);
+               exit(EXIT_FAILURE);
+            }
+            break;
+        case QEMU_TCMU_OPT_DISCARD:
+            if (seen_discard) {
+                error_report("--discard can only be specified once");
+                exit(EXIT_FAILURE);
+            }
+            seen_discard = true;
+            if (bdrv_parse_discard_flags(optarg, &flags) == -1) {
+                error_report("Invalid discard mode `%s'", optarg);
+                exit(EXIT_FAILURE);
+            }
+            break;
+        case QEMU_TCMU_OPT_DETECT_ZEROES:
+            detect_zeroes =
+                qapi_enum_parse(BlockdevDetectZeroesOptions_lookup,
+                                optarg,
+                                BLOCKDEV_DETECT_ZEROES_OPTIONS__MAX,
+                                BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF,
+                                &local_err);
+            if (local_err) {
+                error_reportf_err(local_err,
+                                  "Failed to parse detect_zeroes mode: ");
+                exit(EXIT_FAILURE);
+            }
+            if (detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP &&
+                !(flags & BDRV_O_UNMAP)) {
+                error_report("setting detect-zeroes to unmap is not allowed "
+                             "without setting discard operation to unmap");
+                exit(EXIT_FAILURE);
+            }
+            break;
+        case 'l':
+            if (strstart(optarg, SNAPSHOT_OPT_BASE, NULL)) {
+                sn_opts = qemu_opts_parse_noisily(&internal_snapshot_opts,
+                                                  optarg, false);
+                if (!sn_opts) {
+                    error_report("Failed in parsing snapshot param `%s'",
+                                 optarg);
+                    exit(EXIT_FAILURE);
+                }
+            } else {
+                sn_id_or_name = optarg;
+            }
+            /* fall through */
+        case 'r':
+            flags &= ~BDRV_O_RDWR;
+            break;
+        case 'f':
+            fmt = optarg;
+            break;
+        case 'x':
+            subtype = optarg;
+            break;
+        case 'v':
+            verbose = 1;
+            break;
+        case 'V':
+            version(argv[0]);
+            exit(0);
+            break;
+        case 'h':
+            usage(argv[0]);
+            exit(0);
+            break;
+        case '?':
+            error_report("Try `%s --help' for more information.", argv[0]);
+            exit(EXIT_FAILURE);
+        case QEMU_TCMU_OPT_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                exit(EXIT_FAILURE);
+            }
+        }   break;
+        case QEMU_TCMU_OPT_IMAGE_OPTS:
+            imageOpts = true;
+            break;
+        case 'T':
+            g_free(trace_file);
+            trace_file = trace_opt_parse(optarg);
+            break;
+        }
+    }
+
+    if ((argc - optind) != 1) {
+        error_report("Invalid number of arguments");
+        error_printf("Try `%s --help' for more information.\n", argv[0]);
+        exit(EXIT_FAILURE);
+    }
+
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        exit(EXIT_FAILURE);
+    }
+
+    if (!trace_init_backends()) {
+        exit(1);
+    }
+    trace_init_file(trace_file);
+    qemu_set_log(LOG_TRACE);
+
+    if (qemu_init_main_loop(&local_err)) {
+        error_report_err(local_err);
+        exit(EXIT_FAILURE);
+    }
+    bdrv_init();
+    atexit(bdrv_close_all);
+
+    srcpath = argv[optind];
+    if (imageOpts) {
+        QemuOpts *opts;
+        if (fmt) {
+            error_report("--image-opts and -f are mutually exclusive");
+            exit(EXIT_FAILURE);
+        }
+        opts = qemu_opts_parse_noisily(&file_opts, srcpath, true);
+        if (!opts) {
+            qemu_opts_reset(&file_opts);
+            exit(EXIT_FAILURE);
+        }
+        options = qemu_opts_to_qdict(opts, NULL);
+        qemu_opts_reset(&file_opts);
+        blk = blk_new_open(NULL, NULL, options, flags, &local_err);
+    } else {
+        if (fmt) {
+            options = qdict_new();
+            qdict_put(options, "driver", qstring_from_str(fmt));
+        }
+        blk = blk_new_open(srcpath, NULL, options, flags, &local_err);
+    }
+
+    if (!blk) {
+        error_reportf_err(local_err, "Failed to blk_new_open '%s': ",
+                          argv[optind]);
+        exit(EXIT_FAILURE);
+    }
+    monitor_add_blk(blk, "drive", &error_fatal);
+    bs = blk_bs(blk);
+
+    blk_set_enable_write_cache(blk, !writethrough);
+
+    if (sn_opts) {
+        ret = bdrv_snapshot_load_tmp(bs,
+                                     qemu_opt_get(sn_opts, SNAPSHOT_OPT_ID),
+                                     qemu_opt_get(sn_opts, SNAPSHOT_OPT_NAME),
+                                     &local_err);
+    } else if (sn_id_or_name) {
+        ret = bdrv_snapshot_load_tmp_by_id_or_name(bs, sn_id_or_name,
+                                                   &local_err);
+    }
+    if (ret < 0) {
+        error_reportf_err(local_err, "Failed to load snapshot: ");
+        exit(EXIT_FAILURE);
+    }
+
+    bs->detect_zeroes = detect_zeroes;
+    exp = qemu_tcmu_export(blk, flags & BDRV_O_RDWR, &local_err);
+    if (!exp) {
+        error_reportf_err(local_err, "Failed to create export: ");
+        exit(EXIT_FAILURE);
+    }
+
+    /* now when the initialization is (almost) complete, chdir("/")
+     * to free any busy filesystems */
+    if (chdir("/") < 0) {
+        error_report("Could not chdir to root directory: %s",
+                     strerror(errno));
+        exit(EXIT_FAILURE);
+    }
+
+    state = RUNNING;
+    do {
+        g_main_context_acquire(g_main_context_default());
+        main_loop_wait(starting);
+        g_main_context_release(g_main_context_default());
+        if (starting) {
+            qemu_tcmu_start(subtype, &local_err);
+            if (local_err) {
+                error_report_err(local_err);
+                exit(EXIT_FAILURE);
+            }
+            starting = false;
+        }
+        if (state == TERMINATE) {
+            state = TERMINATING;
+            exp = NULL;
+        }
+    } while (state != TERMINATED);
+
+    blk_unref(blk);
+
+    qemu_opts_del(sn_opts);
+
+    exit(EXIT_SUCCESS);
+}
diff --git a/scsi/Makefile.objs b/scsi/Makefile.objs
new file mode 100644
index 0000000..92e9b30
--- /dev/null
+++ b/scsi/Makefile.objs
@@ -0,0 +1,5 @@
+block-obj-$(CONFIG_TCMU) += tcmu.mo
+
+tcmu.mo-objs       := tcmu.o
+tcmu.mo-cflags     := $(TCMU_CFLAGS)
+tcmu.mo-libs       := $(TCMU_LIBS)
diff --git a/scsi/tcmu.c b/scsi/tcmu.c
new file mode 100644
index 0000000..f70afb7
--- /dev/null
+++ b/scsi/tcmu.c
@@ -0,0 +1,337 @@
+/*
+ *  A TCMU userspace handler for QEMU block drivers.
+ *
+ *  Copyright (C) 2016 Red Hat, Inc.
+ *
+ *  Authors:
+ *      Fam Zheng <famz@redhat.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; under version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "libtcmu.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
+#include "sysemu/block-backend.h"
+#include "block/aio.h"
+#include "block/scsi.h"
+#include "scsi/tcmu.h"
+#include "qemu/main-loop.h"
+#include "qmp-commands.h"
+
+#define TCMU_DEBUG 1
+
+#define DPRINTF(...) do { \
+    printf("[%s:%04d] ", __FILE__, __LINE__); \
+    printf(__VA_ARGS__); \
+} while (0)
+
+typedef struct TCMUExport TCMUExport;
+
+struct TCMUExport {
+    BlockBackend *blk;
+    struct tcmu_device *tcmu_dev;
+    bool writable;
+    QLIST_ENTRY(TCMUExport) next;
+};
+
+typedef struct {
+    struct tcmulib_context *tcmulib_ctx;
+} TCMUHandlerState;
+
+static QLIST_HEAD(, TCMUExport) tcmu_exports =
+    QLIST_HEAD_INITIALIZER(tcmu_exports);
+
+static TCMUHandlerState *handler_state;
+
+#define ASCQ_INVALID_FIELD_IN_CDB 0x2400
+
+typedef struct {
+    struct tcmulib_cmd *cmd;
+    TCMUExport *exp;
+    QEMUIOVector *qiov;
+} TCMURequest;
+
+static void qemu_tcmu_aio_cb(void *opaque, int ret)
+{
+    TCMURequest *req = opaque;
+    DPRINTF("aio cb\n");
+    tcmulib_command_complete(req->exp->tcmu_dev, req->cmd,
+                             ret ? CHECK_CONDITION : GOOD);
+    tcmulib_processing_complete(req->exp->tcmu_dev);
+    g_free(req);
+}
+
+static inline TCMURequest *qemu_tcmu_req_new(TCMUExport *exp,
+                                             struct tcmulib_cmd *cmd,
+                                             QEMUIOVector *qiov)
+{
+    TCMURequest *req = g_new(TCMURequest, 1);
+    *req = (TCMURequest) {
+        .exp = exp,
+        .cmd = cmd,
+        .qiov = qiov,
+    };
+    return req;
+}
+
+static int qemu_tcmu_handle_cmd(TCMUExport *exp, struct tcmulib_cmd *cmd)
+{
+
+    uint8_t *cdb = cmd->cdb;
+    /* TODO: block size? */
+    uint64_t offset = tcmu_get_lba(cdb) << BDRV_SECTOR_BITS;
+    QEMUIOVector *qiov;
+
+    DPRINTF("handle cmd: 0x%x\n", cdb[0]);
+    switch (cdb[0]) {
+    case INQUIRY:
+        return tcmu_emulate_inquiry(exp->tcmu_dev, cdb,
+                                    cmd->iovec, cmd->iov_cnt,
+                                    cmd->sense_buf);
+    case TEST_UNIT_READY:
+        return tcmu_emulate_test_unit_ready(cdb, cmd->iovec, cmd->iov_cnt,
+                                            cmd->sense_buf);
+    case SERVICE_ACTION_IN_16:
+        if (cdb[1] == SAI_READ_CAPACITY_16) {
+            return tcmu_emulate_read_capacity_16(1 << 20,
+                                                 512,
+                                                 cmd->cdb, cmd->iovec,
+                                                 cmd->iov_cnt,
+                                                 cmd->sense_buf);
+        } else {
+            return TCMU_NOT_HANDLED;
+        }
+    case MODE_SENSE:
+    case MODE_SENSE_10:
+        return tcmu_emulate_mode_sense(cdb, cmd->iovec,
+                                       cmd->iov_cnt, cmd->sense_buf);
+    case MODE_SELECT:
+    case MODE_SELECT_10:
+        return tcmu_emulate_mode_select(cdb, cmd->iovec,
+                                        cmd->iov_cnt, cmd->sense_buf);
+    case SYNCHRONIZE_CACHE:
+    case SYNCHRONIZE_CACHE_16:
+        if (cdb[1] & 0x2) {
+            return tcmu_set_sense_data(cmd->sense_buf, ILLEGAL_REQUEST,
+                                       ASCQ_INVALID_FIELD_IN_CDB,
+                                       NULL);
+        } else {
+            blk_aio_flush(exp->blk, qemu_tcmu_aio_cb,
+                          qemu_tcmu_req_new(exp, cmd, NULL));
+            return TCMU_ASYNC_HANDLED;
+        }
+        break;
+    case READ_6:
+    case READ_10:
+    case READ_12:
+    case READ_16:
+        qiov = g_new(QEMUIOVector, 1);
+        qemu_iovec_init_external(qiov, cmd->iovec, cmd->iov_cnt);
+        DPRINTF("read at %ld\n", offset);
+        blk_aio_preadv(exp->blk, offset, qiov, 0, qemu_tcmu_aio_cb,
+                       qemu_tcmu_req_new(exp, cmd, qiov));
+        return TCMU_ASYNC_HANDLED;
+
+    case WRITE_6:
+    case WRITE_10:
+    case WRITE_12:
+    case WRITE_16:
+        qiov = g_new(QEMUIOVector, 1);
+        qemu_iovec_init_external(qiov, cmd->iovec, cmd->iov_cnt);
+        DPRINTF("write at %ld\n", offset);
+        blk_aio_pwritev(exp->blk, offset, qiov, 0, qemu_tcmu_aio_cb,
+                        qemu_tcmu_req_new(exp, cmd, qiov));
+        return TCMU_ASYNC_HANDLED;
+
+    default:
+        DPRINTF("unknown command %x\n", cdb[0]);
+        return TCMU_NOT_HANDLED;
+    }
+}
+
+
+static void qemu_tcmu_dev_event_handler(void *opaque)
+{
+    TCMUExport *exp = opaque;
+    struct tcmulib_cmd *cmd;
+    struct tcmu_device *dev = exp->tcmu_dev;
+
+    tcmulib_processing_start(dev);
+
+    while ((cmd = tcmulib_get_next_command(dev)) != NULL) {
+        int ret = qemu_tcmu_handle_cmd(exp, cmd);
+        if (ret != TCMU_ASYNC_HANDLED) {
+            tcmulib_command_complete(dev, cmd, ret);
+        }
+    }
+
+    tcmulib_processing_complete(dev);
+}
+
+static TCMUExport *qemu_tcmu_lookup(const BlockBackend *blk)
+{
+    TCMUExport *exp;
+
+    QLIST_FOREACH(exp, &tcmu_exports, next) {
+        if (exp->blk == blk) {
+            return exp;
+        }
+    }
+    return NULL;
+}
+static TCMUExport *qemu_tcmu_parse_cfgstr(const char *cfgstr,
+                                          Error **errp);
+
+static bool qemu_tcmu_check_config(const char *cfgstr, char **reason)
+{
+    Error *local_err = NULL;
+
+    qemu_tcmu_parse_cfgstr(cfgstr, &local_err);
+    if (local_err) {
+        *reason = strdup(error_get_pretty(local_err));
+        error_free(local_err);
+        return false;
+    }
+    return true;
+}
+
+static int qemu_tcmu_added(struct tcmu_device *dev)
+{
+    TCMUExport *exp;
+    const char *cfgstr = tcmu_get_dev_cfgstring(dev);
+    Error *local_err = NULL;
+
+    exp = qemu_tcmu_parse_cfgstr(cfgstr, &local_err);
+    if (local_err) {
+        return -1;
+    }
+    exp->tcmu_dev = dev;
+    aio_set_fd_handler(blk_get_aio_context(exp->blk),
+                       tcmu_get_dev_fd(dev),
+                       true, qemu_tcmu_dev_event_handler, NULL, exp);
+    return 0;
+}
+
+static void qemu_tcmu_removed(struct tcmu_device *dev)
+{
+    /* TODO. */
+}
+
+static void qemu_tcmu_master_read(void *opaque)
+{
+    TCMUHandlerState *s = opaque;
+    DPRINTF("tcmu master read\n");
+    tcmulib_master_fd_ready(s->tcmulib_ctx);
+}
+
+static struct tcmulib_handler qemu_tcmu_handler = {
+    .name = "Handler for QEMU block devices",
+    .subtype = NULL, /* Dynamically generated when starting. */
+    .cfg_desc = "Format: device=<name>",
+    .added = qemu_tcmu_added,
+    .removed = qemu_tcmu_removed,
+    .check_config = qemu_tcmu_check_config,
+};
+
+static TCMUExport *qemu_tcmu_parse_cfgstr(const char *cfgstr,
+                                          Error **errp)
+{
+    BlockBackend *blk;
+    const char *dev_str, *device;
+    const char *subtype = qemu_tcmu_handler.subtype;
+    size_t subtype_len;
+    TCMUExport *exp;
+
+    if (!subtype) {
+        error_setg(errp, "TCMU Handler not started");
+    }
+    subtype_len = strlen(subtype);
+    if (strncmp(cfgstr, subtype, subtype_len) ||
+        cfgstr[subtype_len] != '/') {
+        error_report("TCMU: Invalid subtype in device cfgstring: %s", cfgstr);
+        return NULL;
+    }
+    dev_str = &cfgstr[subtype_len + 1];
+    if (dev_str[0] != '@') {
+        error_report("TCMU: Invalid cfgstring format. Must be @<device_name>");
+        return NULL;
+    }
+    device = &dev_str[1];
+
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_setg(errp, "TCMU: Device not found: %s", device);
+        return NULL;
+    }
+    exp = qemu_tcmu_lookup(blk);
+    if (!exp) {
+        error_setg(errp, "TCMU: Device not found: %s", device);
+        return NULL;
+    }
+    return exp;
+}
+
+static void qemu_tcmu_errp(const char *fmt, ...)
+{
+    va_list ap;
+    va_start(ap, fmt);
+    error_vprintf(fmt, ap);
+    va_end(ap);
+}
+
+void qemu_tcmu_start(const char *subtype, Error **errp)
+{
+    int fd;
+
+    DPRINTF("tcmu start\n");
+    if (handler_state) {
+        error_setg(errp, "TCMU handler already started");
+        return;
+    }
+    assert(!qemu_tcmu_handler.subtype);
+    qemu_tcmu_handler.subtype = g_strdup(subtype);
+    handler_state = g_new0(TCMUHandlerState, 1);
+    handler_state->tcmulib_ctx = tcmulib_initialize(&qemu_tcmu_handler, 1,
+                                                    qemu_tcmu_errp);
+    if (!handler_state->tcmulib_ctx) {
+        error_setg(errp, "Failed to initialize tcmulib");
+        goto fail;
+    }
+    fd = tcmulib_get_master_fd(handler_state->tcmulib_ctx);
+    qemu_set_fd_handler(fd, qemu_tcmu_master_read, NULL, handler_state);
+    DPRINTF("register\n");
+    tcmulib_register(handler_state->tcmulib_ctx);
+    return;
+fail:
+    g_free(handler_state);
+    handler_state = NULL;
+}
+
+TCMUExport *qemu_tcmu_export(BlockBackend *blk, bool writable, Error **errp)
+{
+    TCMUExport *exp;
+
+    exp = qemu_tcmu_lookup(blk);
+    if (exp) {
+        error_setg(errp, "Block device already added");
+        return NULL;
+    }
+    exp = g_new0(TCMUExport, 1);
+    exp->blk = blk;
+    blk_ref(blk);
+    exp->writable = writable;
+    QLIST_INSERT_HEAD(&tcmu_exports, exp, next);
+    return exp;
+}
-- 
2.7.4

^ permalink raw reply related	[flat|nested] 9+ messages in thread