From mboxrd@z Thu Jan  1 00:00:00 1970
Content-Type: multipart/mixed; boundary="===============6175766495472488082=="
MIME-Version: 1.0
From: Ming Lin <minggr at gmail.com>
Subject: [SPDK] [RFC PATCH 1/4] changed vhost to support multi-process access
 same target
Date: Thu, 10 May 2018 20:25:51 -0700
Message-ID: <1526009154-5492-2-git-send-email-minggr@gmail.com>
In-Reply-To: 1526009154-5492-1-git-send-email-minggr@gmail.com
List-ID: <spdk@lists.01.org>
To: spdk@lists.01.org

--===============6175766495472488082==
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: quoted-printable

From: Sheng Qiu <sheng.qiu(a)alibaba-inc.com>

Signed-off-by: Sheng Qiu <sheng.qiu(a)alibaba-inc.com>
---
 lib/vhost/vhost.c          |  69 +++++++-
 lib/vhost/vhost_internal.h |   7 +
 lib/vhost/vhost_scsi.c     | 411 +++++++++++++++++++++++++++--------------=
----
 3 files changed, 316 insertions(+), 171 deletions(-)

diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c
index 0d49f59..bd3473e 100644
--- a/lib/vhost/vhost.c
+++ b/lib/vhost/vhost.c
@@ -456,6 +456,29 @@ spdk_vhost_dev_find_id(const char *ctrlr_name)
 	return -1;
 }
 =

+static int
+spdk_vhost_dev_find_id_all(const char *ctrlr_name, int *all_id) {
+        unsigned i, vdev_num =3D 0;
+        size_t dev_dirname_len =3D strlen(dev_dirname);
+
+        if (strncmp(ctrlr_name, dev_dirname, dev_dirname_len) =3D=3D 0) {
+                ctrlr_name +=3D dev_dirname_len;
+        }
+
+        for (i =3D 0; i < MAX_VHOST_DEVICES; i++) {
+                if (g_spdk_vhost_devices[i] =3D=3D NULL) {
+                        continue;
+                }
+
+                if (strcmp(g_spdk_vhost_devices[i]->name, ctrlr_name) =3D=
=3D 0) {
+                        all_id[vdev_num++] =3D i;
+                        assert(vdev_num <=3D MAX_CONNECT_DEVS);
+                }
+        }
+
+        return vdev_num;
+}
+
 struct spdk_vhost_dev *
 spdk_vhost_dev_find(const char *ctrlr_name)
 {
@@ -469,6 +492,22 @@ spdk_vhost_dev_find(const char *ctrlr_name)
 	return g_spdk_vhost_devices[id];
 }
 =

+int =

+spdk_vhost_dev_find_all(const char *ctrlr_name, struct spdk_vhost_dev **vd=
evs)
+{
+        int all_id[MAX_CONNECT_DEVS];
+        int vdev_num =3D 0, i;
+        =

+        vdev_num =3D spdk_vhost_dev_find_id_all(ctrlr_name, all_id);
+        assert(vdev_num <=3D MAX_CONNECT_DEVS);
+ =

+        for (i =3D 0; i < vdev_num; i++) {
+                vdevs[i] =3D g_spdk_vhost_devices[all_id[i]];
+        }
+       =

+        return vdev_num;
+}
+
 static int
 spdk_vhost_parse_core_mask(const char *mask, uint64_t *cpumask)
 {
@@ -509,6 +548,7 @@ spdk_vhost_dev_construct(struct spdk_vhost_dev *vdev, c=
onst char *name, const ch
 	struct stat file_stat;
 	char buf[64];
 	uint64_t cpumask;
+        bool socket_started =3D false; =

 =

 	assert(vdev);
 =

@@ -523,10 +563,11 @@ spdk_vhost_dev_construct(struct spdk_vhost_dev *vdev,=
 const char *name, const ch
 		return -EINVAL;
 	}
 =

+
 	if (spdk_vhost_dev_find(name)) {
-		SPDK_ERRLOG("vhost controller %s already exists.\n", name);
-		return -EEXIST;
-	}
+	        socket_started =3D true;
+        }
+
 =

 	for (ctrlr_num =3D 0; ctrlr_num < MAX_VHOST_DEVICES; ctrlr_num++) {
 		if (g_spdk_vhost_devices[ctrlr_num] =3D=3D NULL) {
@@ -545,6 +586,7 @@ spdk_vhost_dev_construct(struct spdk_vhost_dev *vdev, c=
onst char *name, const ch
 		return -EINVAL;
 	}
 =

+        if (!socket_started) {
 	/* Register vhost driver to handle vhost messages. */
 	if (stat(path, &file_stat) !=3D -1) {
 		if (!S_ISSOCK(file_stat.st_mode)) {
@@ -578,6 +620,7 @@ spdk_vhost_dev_construct(struct spdk_vhost_dev *vdev, c=
onst char *name, const ch
 		SPDK_ERRLOG("Couldn't register callbacks for controller %s\n", name);
 		return -EIO;
 	}
+        }
 =

 	vdev->name =3D strdup(name);
 	vdev->path =3D strdup(path);
@@ -595,6 +638,7 @@ spdk_vhost_dev_construct(struct spdk_vhost_dev *vdev, c=
onst char *name, const ch
 =

 	g_spdk_vhost_devices[ctrlr_num] =3D vdev;
 =

+        if (!socket_started) {
 	if (rte_vhost_driver_start(path) !=3D 0) {
 		spdk_strerror_r(errno, buf, sizeof(buf));
 		SPDK_ERRLOG("Failed to start vhost driver for controller %s (%d): %s\n",=
 name, errno,
@@ -604,6 +648,7 @@ spdk_vhost_dev_construct(struct spdk_vhost_dev *vdev, c=
onst char *name, const ch
 	}
 =

 	SPDK_NOTICELOG("Controller %s: new controller added\n", vdev->name);
+        }
 	return 0;
 }
 =

@@ -935,6 +980,10 @@ start_device(int vid)
 	}
 =

 out:
+        if (rc =3D=3D 0) {
+                assert(vdev->status & 1);                =

+                vdev->status |=3D 2;
+        }
 	pthread_mutex_unlock(&g_spdk_vhost_mutex);
 	return rc;
 }
@@ -1058,14 +1107,18 @@ new_connection(int vid)
 		return -1;
 	}
 =

-	/* since pollers are not running it safe not to use spdk_event here */
 	if (vdev->vid !=3D -1) {
-		SPDK_ERRLOG("Device with vid %d is already connected.\n", vid);
-		pthread_mutex_unlock(&g_spdk_vhost_mutex);
-		return -1;
-	}
+                vdev =3D spdk_vhost_dev_find_next(vdev);
+                if (!vdev) {
+                        SPDK_ERRLOG("Couldn't find unused vdev for vid %d =
to create connection for.\n", vid);
+                        pthread_mutex_unlock(&g_spdk_vhost_mutex);
+                        return -1;
+                }
+        }
 =

 	vdev->vid =3D vid;
+        assert(vdev->status =3D=3D 0);
+        vdev->status |=3D 1;
 	pthread_mutex_unlock(&g_spdk_vhost_mutex);
 	return 0;
 }
diff --git a/lib/vhost/vhost_internal.h b/lib/vhost/vhost_internal.h
index 20931b4..0861287 100644
--- a/lib/vhost/vhost_internal.h
+++ b/lib/vhost/vhost_internal.h
@@ -67,6 +67,8 @@
 =

 #define SPDK_VHOST_IOVS_MAX 128
 =

+#define MAX_CONNECT_DEVS 64
+
 /*
  * Rate at which stats are checked for interrupt coalescing.
  */
@@ -134,6 +136,7 @@ struct spdk_vhost_dev_backend {
 	int (*vhost_remove_controller)(struct spdk_vhost_dev *vdev);
 };
 =

+struct spdk_vhost_scsi_dev;
 struct spdk_vhost_dev {
 	struct rte_vhost_memory *mem;
 	char *name;
@@ -146,6 +149,8 @@ struct spdk_vhost_dev {
 =

 	enum spdk_vhost_dev_type type;
 	const struct spdk_vhost_dev_backend *backend;
+        struct spdk_vhost_scsi_dev *svdev;
+        uint8_t status;
 =

 	uint32_t coalescing_delay_time_base;
 =

@@ -166,6 +171,8 @@ struct spdk_vhost_dev {
 };
 =

 struct spdk_vhost_dev *spdk_vhost_dev_find(const char *ctrlr_name);
+struct spdk_vhost_dev *spdk_vhost_dev_find_next(struct spdk_vhost_dev *vde=
v);
+int spdk_vhost_dev_find_all(const char *ctrlr_name, struct spdk_vhost_dev =
**vdevs);
 void spdk_vhost_dev_mem_register(struct spdk_vhost_dev *vdev);
 void spdk_vhost_dev_mem_unregister(struct spdk_vhost_dev *vdev);
 =

diff --git a/lib/vhost/vhost_scsi.c b/lib/vhost/vhost_scsi.c
index 488d4d1..b9da29e 100644
--- a/lib/vhost/vhost_scsi.c
+++ b/lib/vhost/vhost_scsi.c
@@ -74,7 +74,9 @@ struct spdk_scsi_dev_vhost_state {
 };
 =

 struct spdk_vhost_scsi_dev {
-	struct spdk_vhost_dev vdev;
+	struct spdk_vhost_dev vdev[MAX_CONNECT_DEVS];
+        int conn_vdev_num;
+        bool scsi_dev_initialized;
 	struct spdk_scsi_dev *scsi_dev[SPDK_VHOST_SCSI_CTRLR_MAX_DEVS];
 	struct spdk_scsi_dev_vhost_state scsi_dev_state[SPDK_VHOST_SCSI_CTRLR_MAX=
_DEVS];
 =

@@ -95,6 +97,7 @@ struct spdk_vhost_scsi_task {
 	struct spdk_scsi_dev *scsi_dev;
 =

 	int req_idx;
+        int vdev_idx;
 =

 	/* If set, the task is currently used for I/O processing. */
 	bool used;
@@ -115,6 +118,23 @@ const struct spdk_vhost_dev_backend spdk_vhost_scsi_de=
vice_backend =3D {
 	.vhost_remove_controller =3D spdk_vhost_scsi_dev_remove,
 };
 =

+struct spdk_vhost_dev*
+spdk_vhost_dev_find_next(struct spdk_vhost_dev *vdev)
+{
+        struct spdk_vhost_scsi_dev *svdev =3D vdev->svdev;
+        int i;
+
+        assert(svdev->conn_vdev_num > 0 && svdev->conn_vdev_num < MAX_CONN=
ECT_DEVS);
+        =

+        for (i =3D 0; i < MAX_CONNECT_DEVS; i++) {
+                if (svdev->vdev[i].status =3D=3D 0) {
+                        return &svdev->vdev[i];
+                }
+        }
+
+        return NULL; =

+}
+
 static void
 spdk_vhost_scsi_task_put(struct spdk_vhost_scsi_task *task)
 {
@@ -125,9 +145,10 @@ static void
 spdk_vhost_scsi_task_free_cb(struct spdk_scsi_task *scsi_task)
 {
 	struct spdk_vhost_scsi_task *task =3D SPDK_CONTAINEROF(scsi_task, struct =
spdk_vhost_scsi_task, scsi);
+        int vdev_idx =3D task->vdev_idx;
 =

-	assert(task->svdev->vdev.task_cnt > 0);
-	task->svdev->vdev.task_cnt--;
+	assert(task->svdev->vdev[vdev_idx].task_cnt > 0);
+	task->svdev->vdev[vdev_idx].task_cnt--;
 	task->used =3D false;
 }
 =

@@ -147,16 +168,20 @@ process_removed_devs(struct spdk_vhost_scsi_dev *svde=
v)
 			spdk_scsi_dev_destruct(dev);
 			svdev->scsi_dev[i] =3D NULL;
 			if (state->remove_cb) {
-				state->remove_cb(&svdev->vdev, state->remove_ctx);
+                                /* notify and detach from all impact vdev =
*/
+                                int j;
+                                for (j =3D 0; j < MAX_CONNECT_DEVS; j++) {
+				        state->remove_cb(&svdev->vdev[j], state->remove_ctx);
+                                        SPDK_NOTICELOG("%s: hot-detached d=
evice 'Dev %u'.\n", svdev->vdev[j].name, i);
+                                }
 				state->remove_cb =3D NULL;
 			}
-			SPDK_NOTICELOG("%s: hot-detached device 'Dev %u'.\n", svdev->vdev.name,=
 i);
 		}
 	}
 }
 =

 static void
-eventq_enqueue(struct spdk_vhost_scsi_dev *svdev, unsigned scsi_dev_num, u=
int32_t event,
+eventq_enqueue(struct spdk_vhost_dev* vdev, unsigned scsi_dev_num, uint32_=
t event,
 	       uint32_t reason)
 {
 	struct spdk_vhost_virtqueue *vq;
@@ -167,25 +192,25 @@ eventq_enqueue(struct spdk_vhost_scsi_dev *svdev, uns=
igned scsi_dev_num, uint32_
 	int rc;
 =

 	assert(scsi_dev_num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS);
-	vq =3D &svdev->vdev.virtqueue[VIRTIO_SCSI_EVENTQ];
+	vq =3D &vdev->virtqueue[VIRTIO_SCSI_EVENTQ];
 =

 	if (spdk_vhost_vq_avail_ring_get(vq, &req, 1) !=3D 1) {
 		SPDK_ERRLOG("Controller %s: Failed to send virtio event (no avail ring e=
ntries?).\n",
-			    svdev->vdev.name);
+			    vdev->name);
 		return;
 	}
 =

-	rc =3D spdk_vhost_vq_get_desc(&svdev->vdev, vq, req, &desc, &desc_table, =
&desc_table_size);
+	rc =3D spdk_vhost_vq_get_desc(vdev, vq, req, &desc, &desc_table, &desc_ta=
ble_size);
 	if (rc !=3D 0 || desc->len < sizeof(*desc_ev)) {
 		SPDK_ERRLOG("Controller %s: Invalid eventq descriptor at index %"PRIu16"=
.\n",
-			    svdev->vdev.name, req);
+			    vdev->name, req);
 		goto out;
 	}
 =

-	desc_ev =3D spdk_vhost_gpa_to_vva(&svdev->vdev, desc->addr);
+	desc_ev =3D spdk_vhost_gpa_to_vva(vdev, desc->addr);
 	if (desc_ev =3D=3D NULL) {
 		SPDK_ERRLOG("Controller %s: Eventq descriptor at index %"PRIu16" points =
to unmapped guest memory address %p.\n",
-			    svdev->vdev.name, req, (void *)(uintptr_t)desc->addr);
+			    vdev->name, req, (void *)(uintptr_t)desc->addr);
 		goto out;
 	}
 =

@@ -205,13 +230,14 @@ eventq_enqueue(struct spdk_vhost_scsi_dev *svdev, uns=
igned scsi_dev_num, uint32_
 	req_size =3D sizeof(*desc_ev);
 =

 out:
-	spdk_vhost_vq_used_ring_enqueue(&svdev->vdev, vq, req, req_size);
+	spdk_vhost_vq_used_ring_enqueue(vdev, vq, req, req_size);
 }
 =

 static void
 submit_completion(struct spdk_vhost_scsi_task *task)
 {
-	spdk_vhost_vq_used_ring_enqueue(&task->svdev->vdev, task->vq, task->req_i=
dx,
+        int vdev_idx =3D task->vdev_idx;
+	spdk_vhost_vq_used_ring_enqueue(&task->svdev->vdev[vdev_idx], task->vq, t=
ask->req_idx,
 					task->scsi.data_transferred);
 	SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "Finished task (%p) req_idx=3D%d\n", t=
ask, task->req_idx);
 =

@@ -262,7 +288,8 @@ mgmt_task_submit(struct spdk_vhost_scsi_task *task, enu=
m spdk_scsi_task_func fun
 static void
 invalid_request(struct spdk_vhost_scsi_task *task)
 {
-	spdk_vhost_vq_used_ring_enqueue(&task->svdev->vdev, task->vq, task->req_i=
dx, 0);
+        int vdev_idx =3D task->vdev_idx;
+	spdk_vhost_vq_used_ring_enqueue(&task->svdev->vdev[vdev_idx], task->vq, t=
ask->req_idx, 0);
 	spdk_vhost_scsi_task_put(task);
 =

 	SPDK_DEBUGLOG(SPDK_LOG_VHOST_SCSI, "Invalid request (status=3D%" PRIu8")\=
n",
@@ -299,7 +326,8 @@ spdk_vhost_scsi_task_init_target(struct spdk_vhost_scsi=
_task *task, const __u8 *
 static void
 process_ctrl_request(struct spdk_vhost_scsi_task *task)
 {
-	struct spdk_vhost_dev *vdev =3D &task->svdev->vdev;
+        int vdev_idx =3D task->vdev_idx;
+	struct spdk_vhost_dev *vdev =3D &task->svdev->vdev[vdev_idx];
 	struct vring_desc *desc, *desc_table;
 	struct virtio_scsi_ctrl_tmf_req *ctrl_req;
 	struct virtio_scsi_ctrl_an_resp *an_resp;
@@ -395,7 +423,8 @@ static int
 task_data_setup(struct spdk_vhost_scsi_task *task,
 		struct virtio_scsi_cmd_req **req)
 {
-	struct spdk_vhost_dev *vdev =3D &task->svdev->vdev;
+        int vdev_idx =3D task->vdev_idx;
+	struct spdk_vhost_dev *vdev =3D &task->svdev->vdev[vdev_idx];
 	struct vring_desc *desc, *desc_table;
 	struct iovec *iovs =3D task->iovs;
 	uint16_t iovcnt =3D 0, iovcnt_max =3D SPDK_VHOST_IOVS_MAX;
@@ -557,7 +586,7 @@ process_request(struct spdk_vhost_scsi_task *task)
 }
 =

 static void
-process_controlq(struct spdk_vhost_scsi_dev *svdev, struct spdk_vhost_virt=
queue *vq)
+process_controlq(struct spdk_vhost_scsi_dev *svdev, struct spdk_vhost_virt=
queue *vq, int idx)
 {
 	struct spdk_vhost_scsi_task *task;
 	uint16_t reqs[32];
@@ -567,29 +596,30 @@ process_controlq(struct spdk_vhost_scsi_dev *svdev, s=
truct spdk_vhost_virtqueue
 	for (i =3D 0; i < reqs_cnt; i++) {
 		if (spdk_unlikely(reqs[i] >=3D vq->vring.size)) {
 			SPDK_ERRLOG("%s: invalid entry in avail ring. Buffer '%"PRIu16"' exceed=
s virtqueue size (%"PRIu16")\n",
-				    svdev->vdev.name, reqs[i], vq->vring.size);
-			spdk_vhost_vq_used_ring_enqueue(&svdev->vdev, vq, reqs[i], 0);
+				    svdev->vdev[idx].name, reqs[i], vq->vring.size);
+			spdk_vhost_vq_used_ring_enqueue(&svdev->vdev[idx], vq, reqs[i], 0);
 			continue;
 		}
 =

 		task =3D &((struct spdk_vhost_scsi_task *)vq->tasks)[reqs[i]];
 		if (spdk_unlikely(task->used)) {
 			SPDK_ERRLOG("%s: invalid entry in avail ring. Buffer '%"PRIu16"' is sti=
ll in use!\n",
-				    svdev->vdev.name, reqs[i]);
-			spdk_vhost_vq_used_ring_enqueue(&svdev->vdev, vq, reqs[i], 0);
+				    svdev->vdev[idx].name, reqs[i]);
+			spdk_vhost_vq_used_ring_enqueue(&svdev->vdev[idx], vq, reqs[i], 0);
 			continue;
 		}
 =

-		svdev->vdev.task_cnt++;
+		svdev->vdev[idx].task_cnt++;
 		memset(&task->scsi, 0, sizeof(task->scsi));
 		task->tmf_resp =3D NULL;
 		task->used =3D true;
+                task->vdev_idx =3D idx;
 		process_ctrl_request(task);
 	}
 }
 =

 static void
-process_requestq(struct spdk_vhost_scsi_dev *svdev, struct spdk_vhost_virt=
queue *vq)
+process_requestq(struct spdk_vhost_scsi_dev *svdev, struct spdk_vhost_virt=
queue *vq, int idx)
 {
 	struct spdk_vhost_scsi_task *task;
 	uint16_t reqs[32];
@@ -605,23 +635,24 @@ process_requestq(struct spdk_vhost_scsi_dev *svdev, s=
truct spdk_vhost_virtqueue
 =

 		if (spdk_unlikely(reqs[i] >=3D vq->vring.size)) {
 			SPDK_ERRLOG("%s: request idx '%"PRIu16"' exceeds virtqueue size (%"PRIu=
16").\n",
-				    svdev->vdev.name, reqs[i], vq->vring.size);
-			spdk_vhost_vq_used_ring_enqueue(&svdev->vdev, vq, reqs[i], 0);
+				    svdev->vdev[idx].name, reqs[i], vq->vring.size);
+			spdk_vhost_vq_used_ring_enqueue(&svdev->vdev[idx], vq, reqs[i], 0);
 			continue;
 		}
 =

 		task =3D &((struct spdk_vhost_scsi_task *)vq->tasks)[reqs[i]];
 		if (spdk_unlikely(task->used)) {
 			SPDK_ERRLOG("%s: request with idx '%"PRIu16"' is already pending.\n",
-				    svdev->vdev.name, reqs[i]);
-			spdk_vhost_vq_used_ring_enqueue(&svdev->vdev, vq, reqs[i], 0);
+				    svdev->vdev[idx].name, reqs[i]);
+			spdk_vhost_vq_used_ring_enqueue(&svdev->vdev[idx], vq, reqs[i], 0);
 			continue;
 		}
 =

-		svdev->vdev.task_cnt++;
+		svdev->vdev[idx].task_cnt++;
 		memset(&task->scsi, 0, sizeof(task->scsi));
 		task->resp =3D NULL;
 		task->used =3D true;
+                task->vdev_idx =3D idx;
 		result =3D process_request(task);
 		if (likely(result =3D=3D 0)) {
 			task_submit(task);
@@ -643,12 +674,18 @@ static void
 vdev_mgmt_worker(void *arg)
 {
 	struct spdk_vhost_scsi_dev *svdev =3D arg;
-
-	process_removed_devs(svdev);
-	spdk_vhost_vq_used_signal(&svdev->vdev, &svdev->vdev.virtqueue[VIRTIO_SCS=
I_EVENTQ]);
-
-	process_controlq(svdev, &svdev->vdev.virtqueue[VIRTIO_SCSI_CONTROLQ]);
-	spdk_vhost_vq_used_signal(&svdev->vdev, &svdev->vdev.virtqueue[VIRTIO_SCS=
I_CONTROLQ]);
+        int i;
+
+	process_removed_devs(svdev);        =

+        /* notify all existed vdev */
+        for (i =3D 0; i < MAX_CONNECT_DEVS; i++) {
+                /* only process live vdev */
+                if (svdev->vdev[i].status =3D=3D 3) { =

+	                spdk_vhost_vq_used_signal(&svdev->vdev[i], &svdev->vdev[i=
].virtqueue[VIRTIO_SCSI_EVENTQ]);
+	                process_controlq(svdev, &svdev->vdev[i].virtqueue[VIRTIO_=
SCSI_CONTROLQ], i);
+	                spdk_vhost_vq_used_signal(&svdev->vdev[i], &svdev->vdev[i=
].virtqueue[VIRTIO_SCSI_CONTROLQ]);
+                }
+        }
 }
 =

 static void
@@ -656,12 +693,17 @@ vdev_worker(void *arg)
 {
 	struct spdk_vhost_scsi_dev *svdev =3D arg;
 	uint32_t q_idx;
-
-	for (q_idx =3D VIRTIO_SCSI_REQUESTQ; q_idx < svdev->vdev.num_queues; q_id=
x++) {
-		process_requestq(svdev, &svdev->vdev.virtqueue[q_idx]);
-	}
-
-	spdk_vhost_dev_used_signal(&svdev->vdev);
+        int i;
+
+        for (i =3D 0; i < MAX_CONNECT_DEVS; i++) {
+                /* only process live vdev */
+                if (svdev->vdev[i].status =3D=3D 3) {
+	                for (q_idx =3D VIRTIO_SCSI_REQUESTQ; q_idx < svdev->vdev[=
i].num_queues; q_idx++) {
+		                process_requestq(svdev, &svdev->vdev[i].virtqueue[q_idx]=
, i);
+	                }
+	                spdk_vhost_dev_used_signal(&svdev->vdev[i]);
+                }             =

+        }
 }
 =

 static struct spdk_vhost_scsi_dev *
@@ -677,7 +719,7 @@ to_scsi_dev(struct spdk_vhost_dev *ctrlr)
 		return NULL;
 	}
 =

-	return (struct spdk_vhost_scsi_dev *)ctrlr;
+	return ctrlr->svdev;
 }
 =

 int
@@ -685,20 +727,24 @@ spdk_vhost_scsi_dev_construct(const char *name, const=
 char *cpumask)
 {
 	struct spdk_vhost_scsi_dev *svdev =3D spdk_dma_zmalloc(sizeof(struct spdk=
_vhost_scsi_dev),
 					    SPDK_CACHE_LINE_SIZE, NULL);
-	int rc;
+	int rc, i;
 =

 	if (svdev =3D=3D NULL) {
 		return -ENOMEM;
 	}
+        assert(svdev->conn_vdev_num =3D=3D 0);
 =

 	spdk_vhost_lock();
-	rc =3D spdk_vhost_dev_construct(&svdev->vdev, name, cpumask, SPDK_VHOST_D=
EV_T_SCSI,
-				      &spdk_vhost_scsi_device_backend);
-
-	if (rc) {
-		spdk_dma_free(svdev);
-	}
-
+        for (i =3D 0; i < MAX_CONNECT_DEVS; i++) {
+	        rc =3D spdk_vhost_dev_construct(&svdev->vdev[i], name, cpumask, S=
PDK_VHOST_DEV_T_SCSI,
+				              &spdk_vhost_scsi_device_backend);
+       =

+	        if (rc) {
+		        spdk_dma_free(svdev);
+                        break;
+	        }
+                svdev->vdev[i].svdev =3D svdev;
+        }
 	spdk_vhost_unlock();
 	return rc;
 }
@@ -745,31 +791,35 @@ spdk_vhost_scsi_lun_hotremove(const struct spdk_scsi_=
lun *lun, void *arg)
 {
 	struct spdk_vhost_scsi_dev *svdev =3D arg;
 	const struct spdk_scsi_dev *scsi_dev;
-	unsigned scsi_dev_num;
+	unsigned scsi_dev_num, i;
 =

 	assert(lun !=3D NULL);
 	assert(svdev !=3D NULL);
-	if (!spdk_vhost_dev_has_feature(&svdev->vdev, VIRTIO_SCSI_F_HOTPLUG)) {
-		SPDK_WARNLOG("%s: hotremove is not enabled for this controller.\n", svde=
v->vdev.name);
-		return;
-	}
-
-	scsi_dev =3D spdk_scsi_lun_get_dev(lun);
-	for (scsi_dev_num =3D 0; scsi_dev_num < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; s=
csi_dev_num++) {
-		if (svdev->scsi_dev[scsi_dev_num] =3D=3D scsi_dev) {
-			break;
-		}
-	}
-
-	if (scsi_dev_num =3D=3D SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
-		SPDK_ERRLOG("Dev %s is not a part of vhost scsi controller '%s'.\n",
-			    spdk_scsi_dev_get_name(scsi_dev),
-			    svdev->vdev.name);
-		return;
-	}
-
-	/* remove entire device */
-	spdk_vhost_scsi_dev_remove_tgt(&svdev->vdev, scsi_dev_num, NULL, NULL);
+       =

+        /* find the scsi_dev to be remove */
+        scsi_dev =3D spdk_scsi_lun_get_dev(lun);
+        for (scsi_dev_num =3D 0; scsi_dev_num < SPDK_VHOST_SCSI_CTRLR_MAX_=
DEVS; scsi_dev_num++) {
+                if (svdev->scsi_dev[scsi_dev_num] =3D=3D scsi_dev) {
+                        break;
+                }
+        }
+      =

+        if (scsi_dev_num =3D=3D SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
+                SPDK_ERRLOG("Dev %s is not a part of vhost scsi controller=
 '%s'.\n",
+                            spdk_scsi_dev_get_name(scsi_dev),
+                            svdev->vdev[0].name);
+                return;
+        }
+        =

+        for (i =3D 0; i < MAX_CONNECT_DEVS; i++) {
+	        if (!spdk_vhost_dev_has_feature(&svdev->vdev[i], VIRTIO_SCSI_F_HO=
TPLUG)) {
+		        SPDK_WARNLOG("%s: hotremove is not enabled for this controller.\=
n", svdev->vdev[i].name);
+		        continue;
+	        }
+
+	        /* remove entire device */
+	        spdk_vhost_scsi_dev_remove_tgt(&svdev->vdev[i], scsi_dev_num, NUL=
L, NULL);
+        }
 }
 =

 int
@@ -786,56 +836,65 @@ spdk_vhost_scsi_dev_add_tgt(struct spdk_vhost_dev *vd=
ev, unsigned scsi_tgt_num,
 		return -EINVAL;
 	}
 =

-	if (scsi_tgt_num >=3D SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
-		SPDK_ERRLOG("Controller %d target number too big (max %d)\n", scsi_tgt_n=
um,
-			    SPDK_VHOST_SCSI_CTRLR_MAX_DEVS);
-		return -EINVAL;
-	}
-
-	if (lun_name =3D=3D NULL) {
-		SPDK_ERRLOG("No lun name specified \n");
-		return -EINVAL;
-	} else if (strlen(lun_name) >=3D SPDK_SCSI_DEV_MAX_NAME) {
-		SPDK_ERRLOG("LUN name '%s' too long (max %d).\n", lun_name, SPDK_SCSI_DE=
V_MAX_NAME - 1);
-		return -1;
-	}
+        if (!svdev->scsi_dev_initialized) {
+	        if (scsi_tgt_num >=3D SPDK_VHOST_SCSI_CTRLR_MAX_DEVS) {
+		        SPDK_ERRLOG("Controller %d target number too big (max %d)\n", sc=
si_tgt_num,
+			            SPDK_VHOST_SCSI_CTRLR_MAX_DEVS);
+		        return -EINVAL;
+	        }
+
+	        if (lun_name =3D=3D NULL) {
+		        SPDK_ERRLOG("No lun name specified \n");
+		        return -EINVAL;
+	        } else if (strlen(lun_name) >=3D SPDK_SCSI_DEV_MAX_NAME) {
+		        SPDK_ERRLOG("LUN name '%s' too long (max %d).\n", lun_name, SPDK=
_SCSI_DEV_MAX_NAME - 1);
+		        return -1;
+	        }
+        } =

 =

 	if (vdev->lcore !=3D -1 && !spdk_vhost_dev_has_feature(vdev, VIRTIO_SCSI_=
F_HOTPLUG)) {
 		SPDK_ERRLOG("Controller %s is in use and hotplug is not supported\n", vd=
ev->name);
 		return -ENOTSUP;
 	}
 =

-	if (svdev->scsi_dev[scsi_tgt_num] !=3D NULL) {
-		SPDK_ERRLOG("Controller %s target %u already occupied\n", vdev->name, sc=
si_tgt_num);
-		return -EEXIST;
-	}
-
-	/*
-	 * At this stage only one LUN per target
-	 */
-	snprintf(target_name, sizeof(target_name), "Target %u", scsi_tgt_num);
-	lun_id_list[0] =3D 0;
-	lun_names_list[0] =3D (char *)lun_name;
-
-	svdev->scsi_dev_state[scsi_tgt_num].removed =3D false;
-	svdev->scsi_dev[scsi_tgt_num] =3D spdk_scsi_dev_construct(target_name, lu=
n_names_list, lun_id_list,
-					1,
-					SPDK_SPC_PROTOCOL_IDENTIFIER_SAS, spdk_vhost_scsi_lun_hotremove, svde=
v);
-
-	if (svdev->scsi_dev[scsi_tgt_num] =3D=3D NULL) {
-		SPDK_ERRLOG("Couldn't create spdk SCSI target '%s' using lun device '%s'=
 in controller: %s\n",
-			    target_name, lun_name, vdev->name);
-		return -EINVAL;
-	}
-	spdk_scsi_dev_add_port(svdev->scsi_dev[scsi_tgt_num], 0, "vhost");
+        if (!svdev->scsi_dev_initialized) {
+	        if (svdev->scsi_dev[scsi_tgt_num] !=3D NULL) {
+		        SPDK_ERRLOG("Controller %s target %u already occupied\n", vdev->=
name, scsi_tgt_num);
+		        return -EEXIST;
+	        }
+
+	        /*
+	         * At this stage only one LUN per target
+	         */
+	        snprintf(target_name, sizeof(target_name), "Target %u", scsi_tgt_=
num);
+	        lun_id_list[0] =3D 0;
+	        lun_names_list[0] =3D (char *)lun_name;
+
+	        svdev->scsi_dev_state[scsi_tgt_num].removed =3D false;
+	        svdev->scsi_dev[scsi_tgt_num] =3D spdk_scsi_dev_construct(target_=
name, lun_names_list, lun_id_list,
+					        1,
+					        SPDK_SPC_PROTOCOL_IDENTIFIER_SAS, spdk_vhost_scsi_lun_hotremo=
ve, svdev);
+
+	        if (svdev->scsi_dev[scsi_tgt_num] =3D=3D NULL) {
+		        SPDK_ERRLOG("Couldn't create spdk SCSI target '%s' using lun dev=
ice '%s' in controller: %s\n",
+			            target_name, lun_name, vdev->name);
+		        return -EINVAL;
+	        }
+	        spdk_scsi_dev_add_port(svdev->scsi_dev[scsi_tgt_num], 0, "vhost");
+        }
 =

 	if (vdev->lcore !=3D -1) {
-		spdk_scsi_dev_allocate_io_channels(svdev->scsi_dev[scsi_tgt_num]);
-		eventq_enqueue(svdev, scsi_tgt_num, VIRTIO_SCSI_T_TRANSPORT_RESET, VIRTI=
O_SCSI_EVT_RESET_RESCAN);
+                if (!svdev->scsi_dev_initialized) {
+		        spdk_scsi_dev_allocate_io_channels(svdev->scsi_dev[scsi_tgt_num]=
);
+                }
+		eventq_enqueue(vdev, scsi_tgt_num, VIRTIO_SCSI_T_TRANSPORT_RESET, VIRTIO=
_SCSI_EVT_RESET_RESCAN);
 	}
 =

-	SPDK_NOTICELOG("Controller %s: defined target '%s' using lun '%s'\n",
-		       vdev->name, target_name, lun_name);
+        if (!svdev->scsi_dev_initialized) {
+	        SPDK_NOTICELOG("Controller %s: defined target '%s' using lun '%s'=
\n",
+		               vdev->name, target_name, lun_name);
+                svdev->scsi_dev_initialized =3D true;
+        }
 	return 0;
 }
 =

@@ -864,7 +923,7 @@ spdk_vhost_scsi_dev_remove_tgt(struct spdk_vhost_dev *v=
dev, unsigned scsi_tgt_nu
 		return -ENODEV;
 	}
 =

-	if (svdev->vdev.lcore =3D=3D -1) {
+	if (vdev->lcore =3D=3D -1) {
 		/* controller is not in use, remove dev and exit */
 		spdk_scsi_dev_destruct(scsi_dev);
 		svdev->scsi_dev[scsi_tgt_num] =3D NULL;
@@ -877,13 +936,13 @@ spdk_vhost_scsi_dev_remove_tgt(struct spdk_vhost_dev =
*vdev, unsigned scsi_tgt_nu
 =

 	if (!spdk_vhost_dev_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG)) {
 		SPDK_WARNLOG("%s: 'Target %u' is in use and hot-detach is not enabled fo=
r this controller.\n",
-			     svdev->vdev.name, scsi_tgt_num);
+			     vdev->name, scsi_tgt_num);
 		return -ENOTSUP;
 	}
 =

 	scsi_dev_state =3D &svdev->scsi_dev_state[scsi_tgt_num];
 	if (scsi_dev_state->removed) {
-		SPDK_WARNLOG("%s: 'Target %u' has been already marked to hotremove.\n", =
svdev->vdev.name,
+		SPDK_WARNLOG("%s: 'Target %u' has been already marked to hotremove.\n", =
vdev->name,
 			     scsi_tgt_num);
 		return -EBUSY;
 	}
@@ -891,7 +950,7 @@ spdk_vhost_scsi_dev_remove_tgt(struct spdk_vhost_dev *v=
dev, unsigned scsi_tgt_nu
 	scsi_dev_state->remove_cb =3D cb_fn;
 	scsi_dev_state->remove_ctx =3D cb_arg;
 	scsi_dev_state->removed =3D true;
-	eventq_enqueue(svdev, scsi_tgt_num, VIRTIO_SCSI_T_TRANSPORT_RESET, VIRTIO=
_SCSI_EVT_RESET_REMOVED);
+	eventq_enqueue(vdev, scsi_tgt_num, VIRTIO_SCSI_T_TRANSPORT_RESET, VIRTIO_=
SCSI_EVT_RESET_REMOVED);
 =

 	SPDK_NOTICELOG("%s: queued 'Target %u' for hot-detach.\n", vdev->name, sc=
si_tgt_num);
 	return 0;
@@ -901,8 +960,8 @@ int
 spdk_vhost_scsi_controller_construct(void)
 {
 	struct spdk_conf_section *sp =3D spdk_conf_first_section(NULL);
-	struct spdk_vhost_dev *vdev;
-	int i, dev_num;
+	struct spdk_vhost_dev *vdev[MAX_CONNECT_DEVS];
+	int i, dev_num, vdev_num =3D 0;
 	unsigned ctrlr_num =3D 0;
 	char *lun_name, *tgt_num_str;
 	char *cpumask;
@@ -929,8 +988,9 @@ spdk_vhost_scsi_controller_construct(void)
 			return -1;
 		}
 =

-		vdev =3D spdk_vhost_dev_find(name);
-		assert(vdev);
+                /* multiple vdev supported */
+                vdev_num =3D spdk_vhost_dev_find_all(name, vdev);
+		assert(vdev_num > 0);
 =

 		dev =3D spdk_conf_section_get_nval(sp, "Dev", 0);
 		tgt =3D spdk_conf_section_get_nval(sp, "Target", 0);
@@ -969,10 +1029,13 @@ spdk_vhost_scsi_controller_construct(void)
 				SPDK_ERRLOG("%s: Only one LUN per vhost SCSI device supported\n", name=
);
 				return -1;
 			}
-
-			if (spdk_vhost_scsi_dev_add_tgt(vdev, dev_num, lun_name) < 0) {
-				return -1;
-			}
+                   =

+                        int j;
+                        for (j =3D 0; j < vdev_num; j++) {
+			        if (spdk_vhost_scsi_dev_add_tgt(vdev[j], dev_num, lun_name) < 0=
) {
+				        return -1;
+                                }
+                        }
 		}
 =

 		sp =3D spdk_conf_next_section(sp);
@@ -982,13 +1045,13 @@ spdk_vhost_scsi_controller_construct(void)
 }
 =

 static void
-free_task_pool(struct spdk_vhost_scsi_dev *svdev)
+free_task_pool(struct spdk_vhost_dev *vdev)
 {
 	struct spdk_vhost_virtqueue *vq;
 	uint16_t i;
 =

-	for (i =3D 0; i < svdev->vdev.num_queues; i++) {
-		vq =3D &svdev->vdev.virtqueue[i];
+	for (i =3D 0; i < vdev->num_queues; i++) {
+		vq =3D &vdev->virtqueue[i];
 		if (vq->tasks =3D=3D NULL) {
 			continue;
 		}
@@ -999,7 +1062,7 @@ free_task_pool(struct spdk_vhost_scsi_dev *svdev)
 }
 =

 static int
-alloc_task_pool(struct spdk_vhost_scsi_dev *svdev)
+alloc_task_pool(struct spdk_vhost_scsi_dev *svdev, struct spdk_vhost_dev *=
vdev)
 {
 	struct spdk_vhost_virtqueue *vq;
 	struct spdk_vhost_scsi_task *task;
@@ -1007,23 +1070,23 @@ alloc_task_pool(struct spdk_vhost_scsi_dev *svdev)
 	uint16_t i;
 	uint32_t j;
 =

-	for (i =3D 0; i < svdev->vdev.num_queues; i++) {
-		vq =3D &svdev->vdev.virtqueue[i];
+	for (i =3D 0; i < vdev->num_queues; i++) {
+	        vq =3D &vdev->virtqueue[i];
 		task_cnt =3D vq->vring.size;
 		if (task_cnt > SPDK_VHOST_MAX_VQ_SIZE) {
 			/* sanity check */
 			SPDK_ERRLOG("Controller %s: virtuque %"PRIu16" is too big. (size =3D %"=
PRIu32", max =3D %"PRIu32")\n",
-				    svdev->vdev.name, i, task_cnt, SPDK_VHOST_MAX_VQ_SIZE);
-			free_task_pool(svdev);
+				    vdev->name, i, task_cnt, SPDK_VHOST_MAX_VQ_SIZE);
+			free_task_pool(vdev);
 			return -1;
 		}
 		vq->tasks =3D spdk_dma_zmalloc(sizeof(struct spdk_vhost_scsi_task) * tas=
k_cnt,
 					     SPDK_CACHE_LINE_SIZE, NULL);
 		if (vq->tasks =3D=3D NULL) {
 			SPDK_ERRLOG("Controller %s: failed to allocate %"PRIu32" tasks for virt=
queue %"PRIu16"\n",
-				    svdev->vdev.name, task_cnt, i);
-			free_task_pool(svdev);
-			return -1;
+				    vdev->name, task_cnt, i);
+			free_task_pool(vdev);
+			        return -1;
 		}
 =

 		for (j =3D 0; j < task_cnt; j++) {
@@ -1055,25 +1118,30 @@ spdk_vhost_scsi_start(struct spdk_vhost_dev *vdev, =
void *event_ctx)
 		goto out;
 	}
 =

-	rc =3D alloc_task_pool(svdev);
+	rc =3D alloc_task_pool(svdev, vdev);
 	if (rc !=3D 0) {
 		SPDK_ERRLOG("%s: failed to alloc task pool.\n", vdev->name);
 		goto out;
 	}
 =

-	for (i =3D 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
-		if (svdev->scsi_dev[i] =3D=3D NULL) {
-			continue;
-		}
-		spdk_scsi_dev_allocate_io_channels(svdev->scsi_dev[i]);
-	}
 	SPDK_NOTICELOG("Started poller for vhost controller %s on lcore %d\n", vd=
ev->name, vdev->lcore);
 =

 	spdk_vhost_dev_mem_register(vdev);
 =

-	svdev->requestq_poller =3D spdk_poller_register(vdev_worker, svdev, 0);
-	svdev->mgmt_poller =3D spdk_poller_register(vdev_mgmt_worker, svdev,
-			     MGMT_POLL_PERIOD_US);
+        /* register svdev poller if it's not registered */
+        if (svdev->conn_vdev_num =3D=3D 0) {
+                for (i =3D 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
+                        if (svdev->scsi_dev[i] =3D=3D NULL) {
+                                continue;
+                        }
+                        spdk_scsi_dev_allocate_io_channels(svdev->scsi_dev=
[i]);
+                }
+
+	        svdev->requestq_poller =3D spdk_poller_register(vdev_worker, svde=
v, 0);
+	        svdev->mgmt_poller =3D spdk_poller_register(vdev_mgmt_worker, svd=
ev,
+			             MGMT_POLL_PERIOD_US);
+        }
+        svdev->conn_vdev_num++;
 out:
 	spdk_vhost_dev_backend_event_done(event_ctx, rc);
 	return rc;
@@ -1081,6 +1149,7 @@ out:
 =

 struct spdk_vhost_dev_destroy_ctx {
 	struct spdk_vhost_scsi_dev *svdev;
+        struct spdk_vhost_dev *vdev;
 	struct spdk_poller *poller;
 	void *event_ctx;
 };
@@ -1090,29 +1159,33 @@ destroy_device_poller_cb(void *arg)
 {
 	struct spdk_vhost_dev_destroy_ctx *ctx =3D arg;
 	struct spdk_vhost_scsi_dev *svdev =3D ctx->svdev;
+        struct spdk_vhost_dev *vdev =3D ctx->vdev;
 	uint32_t i;
 =

-	if (svdev->vdev.task_cnt > 0) {
+	if (vdev->task_cnt > 0) {
 		return;
 	}
 =

-
-	for (i =3D 0; i < svdev->vdev.num_queues; i++) {
-		spdk_vhost_vq_used_signal(&svdev->vdev, &svdev->vdev.virtqueue[i]);
-	}
-
-	for (i =3D 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
-		if (svdev->scsi_dev[i] =3D=3D NULL) {
-			continue;
-		}
-		spdk_scsi_dev_free_io_channels(svdev->scsi_dev[i]);
+        assert(vdev->status =3D=3D 3);
+	for (i =3D 0; i < vdev->num_queues; i++) {
+		spdk_vhost_vq_used_signal(vdev, &vdev->virtqueue[i]);
 	}
 =

-	SPDK_NOTICELOG("Stopping poller for vhost controller %s\n", svdev->vdev.n=
ame);
-	spdk_vhost_dev_mem_unregister(&svdev->vdev);
-
-	free_task_pool(svdev);
-
+        if (svdev->conn_vdev_num =3D=3D 0) {
+	        for (i =3D 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
+		        if (svdev->scsi_dev[i] =3D=3D NULL) {
+			        continue;
+		        }
+		        spdk_scsi_dev_free_io_channels(svdev->scsi_dev[i]);
+	        }
+        }
+
+	SPDK_NOTICELOG("Stopping poller for vhost controller %s\n", vdev->name);
+	spdk_vhost_dev_mem_unregister(vdev);
+
+	free_task_pool(vdev);
+        vdev->status =3D 0;
+  =

 	spdk_poller_unregister(&ctx->poller);
 	spdk_vhost_dev_backend_event_done(ctx->event_ctx, 0);
 	spdk_dma_free(ctx);
@@ -1130,6 +1203,10 @@ spdk_vhost_scsi_stop(struct spdk_vhost_dev *vdev, vo=
id *event_ctx)
 		goto err;
 	}
 =

+        /* update active connected vdev count */
+        svdev->conn_vdev_num--;
+        assert(svdev->conn_vdev_num >=3D 0);
+        =

 	destroy_ctx =3D spdk_dma_zmalloc(sizeof(*destroy_ctx), SPDK_CACHE_LINE_SI=
ZE, NULL);
 	if (destroy_ctx =3D=3D NULL) {
 		SPDK_ERRLOG("Failed to alloc memory for destroying device.\n");
@@ -1137,10 +1214,18 @@ spdk_vhost_scsi_stop(struct spdk_vhost_dev *vdev, v=
oid *event_ctx)
 	}
 =

 	destroy_ctx->svdev =3D svdev;
+        destroy_ctx->vdev =3D vdev;
 	destroy_ctx->event_ctx =3D event_ctx;
 =

-	spdk_poller_unregister(&svdev->requestq_poller);
-	spdk_poller_unregister(&svdev->mgmt_poller);
+        /* stop svdev poller only when no live vdev exist */
+        if (svdev->conn_vdev_num =3D=3D 0) {
+                SPDK_NOTICELOG("Stopping svdev, live vdev =3D %d\n", svdev=
->conn_vdev_num);
+	        spdk_poller_unregister(&svdev->requestq_poller);
+	        spdk_poller_unregister(&svdev->mgmt_poller);
+        } else {
+                SPDK_NOTICELOG("Stopping one vdev, live vdev =3D %d\n", sv=
dev->conn_vdev_num);
+        }
+
 	destroy_ctx->poller =3D spdk_poller_register(destroy_device_poller_cb, de=
stroy_ctx,
 			      1000);
 =

-- =

1.9.1


--===============6175766495472488082==--