linux-block.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: John Garry <john.garry@huawei.com>
To: Hannes Reinecke <hare@suse.de>, "axboe@kernel.dk" <axboe@kernel.dk>
Cc: "linux-kernel@vger.kernel.org" <linux-kernel@vger.kernel.org>,
	"linux-block@vger.kernel.org" <linux-block@vger.kernel.org>,
	"ming.lei@redhat.com" <ming.lei@redhat.com>,
	"linux-scsi@vger.kernel.org" <linux-scsi@vger.kernel.org>
Subject: Re: [PATCH RESEND v3 12/13] blk-mq: Use shared tags for shared sbitmap support
Date: Tue, 14 Sep 2021 09:27:04 +0100	[thread overview]
Message-ID: <664ba62f-ccbf-ac0c-548e-e15561b82224@huawei.com> (raw)
In-Reply-To: <05804d91-4392-260d-34f2-618b0d3b3f7f@suse.de>

...

>>    static int blk_mq_init_sched_shared_sbitmap(struct request_queue *queue)
>>    {
>>    	struct blk_mq_tag_set *set = queue->tag_set;
>> -	int alloc_policy = BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags);
>> -	struct blk_mq_hw_ctx *hctx;
>> -	int ret, i;
>>    
>>    	/*
>>    	 * Set initial depth at max so that we don't need to reallocate for
>>    	 * updating nr_requests.
>>    	 */
>> -	ret = blk_mq_init_bitmaps(&queue->sched_bitmap_tags,
>> -				  &queue->sched_breserved_tags,
>> -				  MAX_SCHED_RQ, set->reserved_tags,
>> -				  set->numa_node, alloc_policy);
>> -	if (ret)
>> -		return ret;
>> -
>> -	queue_for_each_hw_ctx(queue, hctx, i) {
>> -		hctx->sched_tags->bitmap_tags =
>> -					&queue->sched_bitmap_tags;
>> -		hctx->sched_tags->breserved_tags =
>> -					&queue->sched_breserved_tags;
>> -	}
>> +	queue->shared_sbitmap_tags = blk_mq_alloc_map_and_rqs(set,
>> +						BLK_MQ_NO_HCTX_IDX,
>> +						MAX_SCHED_RQ);
>> +	if (!queue->shared_sbitmap_tags)
>> +		return -ENOMEM;
>>    
> 
> Any particular reason why the 'shared_sbitmap_tags' pointer is added to
> the request queue and not the tagset?
> Everything else is located there, so I would have found it more logical
> to add the 'shared_sbitmap_tags' pointer to the tagset, not the queue ...
> 

We already have it for both. Since commit d97e594c5166 ("blk-mq: Use 
request queue-wide tags for tagset-wide sbitmap"), we have a "shared 
sbitmap" per tagset and per request queue. If you check d97e594c5166 
then it should explain the reason.

>>    	blk_mq_tag_update_sched_shared_sbitmap(queue);
>>    
>>    	return 0;
>>    }
>>    
>> -static void blk_mq_exit_sched_shared_sbitmap(struct request_queue *queue)
>> -{
>> -	sbitmap_queue_free(&queue->sched_bitmap_tags);
>> -	sbitmap_queue_free(&queue->sched_breserved_tags);
>> -}
>> -
>>    int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e)
>>    {
>> +	unsigned int i, flags = q->tag_set->flags;
>>    	struct blk_mq_hw_ctx *hctx;
>>    	struct elevator_queue *eq;
>> -	unsigned int i;
>>    	int ret;
>>    
>>    	if (!e) {
>> @@ -598,21 +596,21 @@ int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e)
>>    	q->nr_requests = 2 * min_t(unsigned int, q->tag_set->queue_depth,
>>    				   BLKDEV_DEFAULT_RQ);
>>    
>> -	queue_for_each_hw_ctx(q, hctx, i) {
>> -		ret = blk_mq_sched_alloc_map_and_rqs(q, hctx, i);
>> +	if (blk_mq_is_sbitmap_shared(flags)) {
>> +		ret = blk_mq_init_sched_shared_sbitmap(q); >   		if (ret)
>> -			goto err_free_map_and_rqs;
>> +			return ret;
>>    	}
>>    
>> -	if (blk_mq_is_sbitmap_shared(q->tag_set->flags)) {
>> -		ret = blk_mq_init_sched_shared_sbitmap(q);
>> +	queue_for_each_hw_ctx(q, hctx, i) {
>> +		ret = blk_mq_sched_alloc_map_and_rqs(q, hctx, i);
>>    		if (ret)
>>    			goto err_free_map_and_rqs;
>>    	}
>>    
>>    	ret = e->ops.init_sched(q, e);
>>    	if (ret)
>> -		goto err_free_sbitmap;
>> +		goto err_free_map_and_rqs;
>>    
>>    	blk_mq_debugfs_register_sched(q);
>>    
>> @@ -632,12 +630,10 @@ int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e)
>>    
>>    	return 0;
>>    
>> -err_free_sbitmap:
>> -	if (blk_mq_is_sbitmap_shared(q->tag_set->flags))
>> -		blk_mq_exit_sched_shared_sbitmap(q);
>> -	blk_mq_sched_free_rqs(q);
>>    err_free_map_and_rqs:
>> -	blk_mq_sched_tags_teardown(q);
>> +	blk_mq_sched_free_rqs(q);
>> +	blk_mq_sched_tags_teardown(q, flags);
>> +
>>    	q->elevator = NULL;
>>    	return ret;
>>    }
>> @@ -651,9 +647,15 @@ void blk_mq_sched_free_rqs(struct request_queue *q)
>>    	struct blk_mq_hw_ctx *hctx;
>>    	int i;
>>    
>> -	queue_for_each_hw_ctx(q, hctx, i) {
>> -		if (hctx->sched_tags)
>> -			blk_mq_free_rqs(q->tag_set, hctx->sched_tags, i);
>> +	if (blk_mq_is_sbitmap_shared(q->tag_set->flags)) {
>> +		blk_mq_free_rqs(q->tag_set, q->shared_sbitmap_tags,
>> +				BLK_MQ_NO_HCTX_IDX);
> 
> 'if (q->shared_sbitmap_tags)'
> 
> would be more obvious here ...

I suppose so. I am just doing it this way for consistency.

> 
>> +	} else {
>> +		queue_for_each_hw_ctx(q, hctx, i) {
>> +			if (hctx->sched_tags)
>> +				blk_mq_free_rqs(q->tag_set,
>> +						hctx->sched_tags, i);
>> +		}
>>    	}
>>    }
>>    

...

>>    };
>> @@ -432,6 +429,8 @@ enum {
>>    	((policy & ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) \
>>    		<< BLK_MQ_F_ALLOC_POLICY_START_BIT)
>>    
>> +#define BLK_MQ_NO_HCTX_IDX	(-1U)
>> +
>>    struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata,
>>    		struct lock_class_key *lkclass);
>>    #define blk_mq_alloc_disk(set, queuedata)				\
>> diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
>> index 4baf9435232d..17e50e5ef47b 100644
>> --- a/include/linux/blkdev.h
>> +++ b/include/linux/blkdev.h
>> @@ -459,8 +459,7 @@ struct request_queue {
>>    
>>    	atomic_t		nr_active_requests_shared_sbitmap;
>>    
>> -	struct sbitmap_queue	sched_bitmap_tags;
>> -	struct sbitmap_queue	sched_breserved_tags;
>> +	struct blk_mq_tags	*shared_sbitmap_tags;
>>    
>>    	struct list_head	icq_list;
>>    #ifdef CONFIG_BLK_CGROUP
>>
> Why the double shared_sbitmap_tags pointer in struct request_queue and
> struct tag_set? To my knowledge there's a 1:1 relationship between
> request_queue and tag_set, so where's the point?
> 

As above, we also added a shared sbitmap per request queue.

The reason being that for "shared sbitmap" support, the request queue 
total depth should not grow for an increase in the number of HW queues.

Thanks,
John


  reply	other threads:[~2021-09-14  8:23 UTC|newest]

Thread overview: 30+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-09-13 15:12 [PATCH RESEND v3 00/13] blk-mq: Reduce static requests memory footprint for shared sbitmap John Garry
2021-09-13 15:12 ` [PATCH RESEND v3 01/13] blk-mq: Change rqs check in blk_mq_free_rqs() John Garry
2021-09-14  5:35   ` Hannes Reinecke
2021-09-13 15:12 ` [PATCH RESEND v3 02/13] block: Rename BLKDEV_MAX_RQ -> BLKDEV_DEFAULT_RQ John Garry
2021-09-14  5:36   ` Hannes Reinecke
2021-09-13 15:12 ` [PATCH RESEND v3 03/13] blk-mq: Relocate shared sbitmap resize in blk_mq_update_nr_requests() John Garry
2021-09-14  5:37   ` Hannes Reinecke
2021-09-13 15:12 ` [PATCH RESEND v3 04/13] blk-mq: Invert check " John Garry
2021-09-14  5:38   ` Hannes Reinecke
2021-09-13 15:12 ` [PATCH RESEND v3 05/13] blk-mq-sched: Rename blk_mq_sched_alloc_{tags -> map_and_rqs}() John Garry
2021-09-14  5:40   ` Hannes Reinecke
2021-09-14  8:10     ` John Garry
2021-09-13 15:12 ` [PATCH RESEND v3 06/13] blk-mq-sched: Rename blk_mq_sched_free_{requests -> rqs}() John Garry
2021-09-14  5:41   ` Hannes Reinecke
2021-09-13 15:12 ` [PATCH RESEND v3 07/13] blk-mq: Pass driver tags to blk_mq_clear_rq_mapping() John Garry
2021-09-14  5:44   ` Hannes Reinecke
2021-09-13 15:12 ` [PATCH RESEND v3 08/13] blk-mq: Don't clear driver tags own mapping John Garry
2021-09-14  5:45   ` Hannes Reinecke
2021-09-13 15:12 ` [PATCH RESEND v3 09/13] blk-mq: Add blk_mq_tag_update_sched_shared_sbitmap() John Garry
2021-09-14  5:46   ` Hannes Reinecke
2021-09-13 15:12 ` [PATCH RESEND v3 10/13] blk-mq: Add blk_mq_alloc_map_and_rqs() John Garry
2021-09-14  5:48   ` Hannes Reinecke
2021-09-13 15:12 ` [PATCH RESEND v3 11/13] blk-mq: Refactor and rename blk_mq_free_map_and_{requests->rqs}() John Garry
2021-09-14  5:50   ` Hannes Reinecke
2021-09-13 15:12 ` [PATCH RESEND v3 12/13] blk-mq: Use shared tags for shared sbitmap support John Garry
2021-09-14  6:04   ` Hannes Reinecke
2021-09-14  8:27     ` John Garry [this message]
2021-09-13 15:12 ` [PATCH RESEND v3 13/13] blk-mq: Stop using pointers for blk_mq_tags bitmap tags John Garry
2021-09-14  6:05   ` Hannes Reinecke
2021-09-20  7:45 ` [PATCH RESEND v3 00/13] blk-mq: Reduce static requests memory footprint for shared sbitmap John Garry

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=664ba62f-ccbf-ac0c-548e-e15561b82224@huawei.com \
    --to=john.garry@huawei.com \
    --cc=axboe@kernel.dk \
    --cc=hare@suse.de \
    --cc=linux-block@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-scsi@vger.kernel.org \
    --cc=ming.lei@redhat.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).