Re: [PATCH 7/9] scsi: hisi_sas_v3: convert private reply queue to blk-mq hw queue

[Hannes Reinecke <hare@suse.de>]

On 5/31/19 4:27 AM, Ming Lei wrote:
> SCSI's reply qeueue is very similar with blk-mq's hw queue, both
> assigned by IRQ vector, so map te private reply queue into blk-mq's hw
> queue via .host_tagset.
> 
> Then the private reply mapping can be removed.
> 
> Another benefit is that the request/irq lost issue may be solved in
> generic approach because managed IRQ may be shutdown during CPU
> hotplug.
> 
> Signed-off-by: Ming Lei <ming.lei@redhat.com>
> ---
>  drivers/scsi/hisi_sas/hisi_sas.h       |  2 +-
>  drivers/scsi/hisi_sas/hisi_sas_main.c  | 36 ++++++++++----------
>  drivers/scsi/hisi_sas/hisi_sas_v3_hw.c | 46 +++++++++-----------------
>  3 files changed, 36 insertions(+), 48 deletions(-)
> 
> diff --git a/drivers/scsi/hisi_sas/hisi_sas.h b/drivers/scsi/hisi_sas/hisi_sas.h
> index fc87994b5d73..3d48848dbde7 100644
> --- a/drivers/scsi/hisi_sas/hisi_sas.h
> +++ b/drivers/scsi/hisi_sas/hisi_sas.h
> @@ -26,6 +26,7 @@
>  #include <linux/platform_device.h>
>  #include <linux/property.h>
>  #include <linux/regmap.h>
> +#include <linux/blk-mq-pci.h>
>  #include <scsi/sas_ata.h>
>  #include <scsi/libsas.h>
>  
> @@ -378,7 +379,6 @@ struct hisi_hba {
>  	u32 intr_coal_count;	/* Interrupt count to coalesce */
>  
>  	int cq_nvecs;
> -	unsigned int *reply_map;
>  
>  	/* debugfs memories */
>  	u32 *debugfs_global_reg;
> diff --git a/drivers/scsi/hisi_sas/hisi_sas_main.c b/drivers/scsi/hisi_sas/hisi_sas_main.c
> index 8a7feb8ed8d6..a1c1f30b9fdb 100644
> --- a/drivers/scsi/hisi_sas/hisi_sas_main.c
> +++ b/drivers/scsi/hisi_sas/hisi_sas_main.c
> @@ -441,6 +441,19 @@ static int hisi_sas_dif_dma_map(struct hisi_hba *hisi_hba,
>  	return rc;
>  }
>  
> +static struct scsi_cmnd *sas_task_to_scsi_cmd(struct sas_task *task)
> +{
> +	if (!task->uldd_task)
> +		return NULL;
> +
> +	if (dev_is_sata(task->dev)) {
> +		struct ata_queued_cmd *qc = task->uldd_task;
> +		return qc->scsicmd;
> +	} else {
> +		return task->uldd_task;
> +	}
> +}
> +
>  static int hisi_sas_task_prep(struct sas_task *task,
>  			      struct hisi_sas_dq **dq_pointer,
>  			      bool is_tmf, struct hisi_sas_tmf_task *tmf,
> @@ -459,6 +472,7 @@ static int hisi_sas_task_prep(struct sas_task *task,
>  	struct hisi_sas_dq *dq;
>  	unsigned long flags;
>  	int wr_q_index;
> +	struct scsi_cmnd *scsi_cmnd;
>  
>  	if (DEV_IS_GONE(sas_dev)) {
>  		if (sas_dev)
> @@ -471,9 +485,10 @@ static int hisi_sas_task_prep(struct sas_task *task,
>  		return -ECOMM;
>  	}
>  
> -	if (hisi_hba->reply_map) {
> -		int cpu = raw_smp_processor_id();
> -		unsigned int dq_index = hisi_hba->reply_map[cpu];
> +	scsi_cmnd = sas_task_to_scsi_cmd(task);
> +	if (hisi_hba->shost->hostt->host_tagset) {
> +		unsigned int dq_index = scsi_cmnd_hctx_index(
> +				hisi_hba->shost, scsi_cmnd);
>  
>  		*dq_pointer = dq = &hisi_hba->dq[dq_index];
>  	} else {
> @@ -503,21 +518,8 @@ static int hisi_sas_task_prep(struct sas_task *task,
>  
>  	if (hisi_hba->hw->slot_index_alloc)
>  		rc = hisi_hba->hw->slot_index_alloc(hisi_hba, device);
> -	else {
> -		struct scsi_cmnd *scsi_cmnd = NULL;
> -
> -		if (task->uldd_task) {
> -			struct ata_queued_cmd *qc;
> -
> -			if (dev_is_sata(device)) {
> -				qc = task->uldd_task;
> -				scsi_cmnd = qc->scsicmd;
> -			} else {
> -				scsi_cmnd = task->uldd_task;
> -			}
> -		}
> +	else
>  		rc  = hisi_sas_slot_index_alloc(hisi_hba, scsi_cmnd);
> -	}
>  	if (rc < 0)
>  		goto err_out_dif_dma_unmap;
>  
> diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
> index 49620c2411df..063e50e5b30c 100644
> --- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
> +++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
> @@ -2344,30 +2344,6 @@ static irqreturn_t cq_interrupt_v3_hw(int irq_no, void *p)
>  	return IRQ_HANDLED;
>  }
>  
> -static void setup_reply_map_v3_hw(struct hisi_hba *hisi_hba, int nvecs)
> -{
> -	const struct cpumask *mask;
> -	int queue, cpu;
> -
> -	for (queue = 0; queue < nvecs; queue++) {
> -		struct hisi_sas_cq *cq = &hisi_hba->cq[queue];
> -
> -		mask = pci_irq_get_affinity(hisi_hba->pci_dev, queue +
> -					    BASE_VECTORS_V3_HW);
> -		if (!mask)
> -			goto fallback;
> -		cq->pci_irq_mask = mask;
> -		for_each_cpu(cpu, mask)
> -			hisi_hba->reply_map[cpu] = queue;
> -	}
> -	return;
> -
> -fallback:
> -	for_each_possible_cpu(cpu)
> -		hisi_hba->reply_map[cpu] = cpu % hisi_hba->queue_count;
> -	/* Don't clean all CQ masks */
> -}
> -
>  static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba)
>  {
>  	struct device *dev = hisi_hba->dev;
> @@ -2383,11 +2359,6 @@ static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba)
>  
>  		min_msi = MIN_AFFINE_VECTORS_V3_HW;
>  
> -		hisi_hba->reply_map = devm_kcalloc(dev, nr_cpu_ids,
> -						   sizeof(unsigned int),
> -						   GFP_KERNEL);
> -		if (!hisi_hba->reply_map)
> -			return -ENOMEM;
>  		vectors = pci_alloc_irq_vectors_affinity(hisi_hba->pci_dev,
>  							 min_msi, max_msi,
>  							 PCI_IRQ_MSI |
> @@ -2395,7 +2366,6 @@ static int interrupt_init_v3_hw(struct hisi_hba *hisi_hba)
>  							 &desc);
>  		if (vectors < 0)
>  			return -ENOENT;
> -		setup_reply_map_v3_hw(hisi_hba, vectors - BASE_VECTORS_V3_HW);
>  	} else {
>  		min_msi = max_msi;
>  		vectors = pci_alloc_irq_vectors(hisi_hba->pci_dev, min_msi,
> @@ -2896,6 +2866,18 @@ static void debugfs_snapshot_restore_v3_hw(struct hisi_hba *hisi_hba)
>  	clear_bit(HISI_SAS_REJECT_CMD_BIT, &hisi_hba->flags);
>  }
>  
> +static int hisi_sas_map_queues(struct Scsi_Host *shost)
> +{
> +	struct hisi_hba *hisi_hba = shost_priv(shost);
> +	struct blk_mq_queue_map *qmap = &shost->tag_set.map[HCTX_TYPE_DEFAULT];
> +
> +	if (auto_affine_msi_experimental)
> +		return blk_mq_pci_map_queues(qmap, hisi_hba->pci_dev,
> +				BASE_VECTORS_V3_HW);
> +	else
> +		return blk_mq_map_queues(qmap);
> +}
> +
>  static struct scsi_host_template sht_v3_hw = {
>  	.name			= DRV_NAME,
>  	.module			= THIS_MODULE,

As mentioned, we should be using a common function here.

> @@ -2906,6 +2888,8 @@ static struct scsi_host_template sht_v3_hw = {
>  	.scan_start		= hisi_sas_scan_start,
>  	.change_queue_depth	= sas_change_queue_depth,
>  	.bios_param		= sas_bios_param,
> +	.map_queues		= hisi_sas_map_queues,
> +	.host_tagset		= 1,
>  	.this_id		= -1,
>  	.sg_tablesize		= HISI_SAS_SGE_PAGE_CNT,
>  	.sg_prot_tablesize	= HISI_SAS_SGE_PAGE_CNT,
> @@ -3092,6 +3076,8 @@ hisi_sas_v3_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>  	if (hisi_sas_debugfs_enable)
>  		hisi_sas_debugfs_init(hisi_hba);
>  
> +	shost->nr_hw_queues = hisi_hba->cq_nvecs;
> +
>  	rc = scsi_add_host(shost, dev);
>  	if (rc)
>  		goto err_out_ha;
> 
Well, I'd rather see the v3 hardware converted to 'real' blk-mq first;
the hardware itself is pretty much multiqueue already, so we should be
better off converting it to blk-mq.

Cheers,

Hannes
-- 
Dr. Hannes Reinecke		   Teamlead Storage & Networking
hare@suse.de			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 Nürnberg
GF: Felix Imendörffer, Mary Higgins, Sri Rasiah
HRB 21284 (AG Nürnberg)