From 2ac2af2b1316adc934d0e699985567ded595fe26 Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Thu, 26 Aug 2021 22:40:39 +0800 Subject: [PATCH] block/mq-deadline: Speed up the dispatch of low-priority requests dd_queued() traverses the percpu variable for summation. The more cores, the higher the performance overhead. I currently have a 128-core board and this function takes 2.5 us. If the number of high-priority requests is small and the number of low- and medium-priority requests is large, the performance impact is significant. Let's maintain a non-percpu member variable 'nr_queued', which is incremented by 1 immediately following "inserted++" and decremented by 1 immediately following "completed++". Because both the judgment dd_queued() in dd_dispatch_request() and operation "inserted++" in dd_insert_request() are protected by dd->lock, lock protection needs to be added only in dd_finish_request(), which is unlikely to cause significant performance side effects. Tested on my 128-core board with two ssd disks. fio bs=4k rw=read iodepth=128 cpus_allowed=0-95 Before: [183K/0/0 iops] [172K/0/0 iops] After: [258K/0/0 iops] [258K/0/0 iops] Fixes: fb926032b320 ("block/mq-deadline: Prioritize high-priority requests") Signed-off-by: Zhen Lei Signed-off-by: Damien Le Moal --- block/mq-deadline.c | 81 ++++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 38 deletions(-) diff --git a/block/mq-deadline.c b/block/mq-deadline.c index a09761cbdf12..0d879f3ff340 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -79,6 +79,7 @@ struct dd_per_prio { struct list_head fifo_list[DD_DIR_COUNT]; /* Next request in FIFO order. Read, write or both are NULL. */ struct request *next_rq[DD_DIR_COUNT]; + unsigned int nr_queued; }; struct deadline_data { @@ -106,7 +107,6 @@ struct deadline_data { int aging_expire; spinlock_t lock; - spinlock_t zone_lock; }; /* Count one event of type 'event_type' and with I/O priority 'prio' */ @@ -276,10 +276,12 @@ deadline_move_request(struct deadline_data *dd, struct dd_per_prio *per_prio, deadline_remove_request(rq->q, per_prio, rq); } -/* Number of requests queued for a given priority level. */ -static u32 dd_queued(struct deadline_data *dd, enum dd_prio prio) +/* + * Number of requests queued for a given priority level. + */ +static inline u32 dd_queued(struct deadline_data *dd, enum dd_prio prio) { - return dd_sum(dd, inserted, prio) - dd_sum(dd, completed, prio); + return dd->per_prio[prio].nr_queued; } /* @@ -309,7 +311,6 @@ deadline_fifo_request(struct deadline_data *dd, struct dd_per_prio *per_prio, enum dd_data_dir data_dir) { struct request *rq; - unsigned long flags; if (list_empty(&per_prio->fifo_list[data_dir])) return NULL; @@ -322,16 +323,12 @@ deadline_fifo_request(struct deadline_data *dd, struct dd_per_prio *per_prio, * Look for a write request that can be dispatched, that is one with * an unlocked target zone. */ - spin_lock_irqsave(&dd->zone_lock, flags); list_for_each_entry(rq, &per_prio->fifo_list[DD_WRITE], queuelist) { if (blk_req_can_dispatch_to_zone(rq)) - goto out; + return rq; } - rq = NULL; -out: - spin_unlock_irqrestore(&dd->zone_lock, flags); - return rq; + return NULL; } /* @@ -343,7 +340,6 @@ deadline_next_request(struct deadline_data *dd, struct dd_per_prio *per_prio, enum dd_data_dir data_dir) { struct request *rq; - unsigned long flags; rq = per_prio->next_rq[data_dir]; if (!rq) @@ -356,15 +352,13 @@ deadline_next_request(struct deadline_data *dd, struct dd_per_prio *per_prio, * Look for a write request that can be dispatched, that is one with * an unlocked target zone. */ - spin_lock_irqsave(&dd->zone_lock, flags); while (rq) { if (blk_req_can_dispatch_to_zone(rq)) - break; + return rq; rq = deadline_latter_request(rq); } - spin_unlock_irqrestore(&dd->zone_lock, flags); - return rq; + return NULL; } /* @@ -497,8 +491,10 @@ static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx) const u64 now_ns = ktime_get_ns(); struct request *rq = NULL; enum dd_prio prio; + unsigned long flags; + + spin_lock_irqsave(&dd->lock, flags); - spin_lock(&dd->lock); /* * Start with dispatching requests whose deadline expired more than * aging_expire jiffies ago. @@ -520,7 +516,7 @@ static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx) } unlock: - spin_unlock(&dd->lock); + spin_unlock_irqrestore(&dd->lock, flags); return rq; } @@ -622,7 +618,6 @@ static int dd_init_sched(struct request_queue *q, struct elevator_type *e) dd->fifo_batch = fifo_batch; dd->aging_expire = aging_expire; spin_lock_init(&dd->lock); - spin_lock_init(&dd->zone_lock); q->elevator = eq; return 0; @@ -675,10 +670,11 @@ static bool dd_bio_merge(struct request_queue *q, struct bio *bio, struct deadline_data *dd = q->elevator->elevator_data; struct request *free = NULL; bool ret; + unsigned long flags; - spin_lock(&dd->lock); + spin_lock_irqsave(&dd->lock, flags); ret = blk_mq_sched_try_merge(q, bio, nr_segs, &free); - spin_unlock(&dd->lock); + spin_unlock_irqrestore(&dd->lock, flags); if (free) blk_mq_free_request(free); @@ -690,7 +686,7 @@ static bool dd_bio_merge(struct request_queue *q, struct bio *bio, * add rq to rbtree and fifo */ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, - bool at_head) + bool at_head, struct list_head *free) { struct request_queue *q = hctx->queue; struct deadline_data *dd = q->elevator->elevator_data; @@ -699,7 +695,6 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, u8 ioprio_class = IOPRIO_PRIO_CLASS(ioprio); struct dd_per_prio *per_prio; enum dd_prio prio; - LIST_HEAD(free); lockdep_assert_held(&dd->lock); @@ -712,14 +707,14 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, prio = ioprio_class_to_prio[ioprio_class]; dd_count(dd, inserted, prio); - if (blk_mq_sched_try_insert_merge(q, rq, &free)) { - blk_mq_free_requests(&free); + if (blk_mq_sched_try_insert_merge(q, rq, free)) return; - } + + per_prio = &dd->per_prio[prio]; + per_prio->nr_queued++; trace_block_rq_insert(rq); - per_prio = &dd->per_prio[prio]; if (at_head) { list_add(&rq->queuelist, &per_prio->dispatch); } else { @@ -747,16 +742,23 @@ static void dd_insert_requests(struct blk_mq_hw_ctx *hctx, { struct request_queue *q = hctx->queue; struct deadline_data *dd = q->elevator->elevator_data; + unsigned long flags; + LIST_HEAD(free); - spin_lock(&dd->lock); + spin_lock_irqsave(&dd->lock, flags); while (!list_empty(list)) { struct request *rq; rq = list_first_entry(list, struct request, queuelist); list_del_init(&rq->queuelist); - dd_insert_request(hctx, rq, at_head); + dd_insert_request(hctx, rq, at_head, &free); + if (!list_empty(&free)) { + spin_unlock_irqrestore(&dd->lock, flags); + blk_mq_free_requests(&free); + spin_lock_irqsave(&dd->lock, flags); + } } - spin_unlock(&dd->lock); + spin_unlock_irqrestore(&dd->lock, flags); } /* @@ -790,18 +792,21 @@ static void dd_finish_request(struct request *rq) const u8 ioprio_class = dd_rq_ioclass(rq); const enum dd_prio prio = ioprio_class_to_prio[ioprio_class]; struct dd_per_prio *per_prio = &dd->per_prio[prio]; + unsigned long flags; dd_count(dd, completed, prio); - if (blk_queue_is_zoned(q)) { - unsigned long flags; + spin_lock_irqsave(&dd->lock, flags); - spin_lock_irqsave(&dd->zone_lock, flags); + per_prio->nr_queued--; + + if (blk_queue_is_zoned(q)) { blk_req_zone_write_unlock(rq); if (!list_empty(&per_prio->fifo_list[DD_WRITE])) blk_mq_sched_mark_restart_hctx(rq->mq_hctx); - spin_unlock_irqrestore(&dd->zone_lock, flags); } + + spin_unlock_irqrestore(&dd->lock, flags); } static bool dd_has_work_for_prio(struct dd_per_prio *per_prio) @@ -899,7 +904,7 @@ static void *deadline_##name##_fifo_start(struct seq_file *m, \ struct deadline_data *dd = q->elevator->elevator_data; \ struct dd_per_prio *per_prio = &dd->per_prio[prio]; \ \ - spin_lock(&dd->lock); \ + spin_lock_irq(&dd->lock); \ return seq_list_start(&per_prio->fifo_list[data_dir], *pos); \ } \ \ @@ -919,7 +924,7 @@ static void deadline_##name##_fifo_stop(struct seq_file *m, void *v) \ struct request_queue *q = m->private; \ struct deadline_data *dd = q->elevator->elevator_data; \ \ - spin_unlock(&dd->lock); \ + spin_unlock_irq(&dd->lock); \ } \ \ static const struct seq_operations deadline_##name##_fifo_seq_ops = { \ @@ -1015,7 +1020,7 @@ static void *deadline_dispatch##prio##_start(struct seq_file *m, \ struct deadline_data *dd = q->elevator->elevator_data; \ struct dd_per_prio *per_prio = &dd->per_prio[prio]; \ \ - spin_lock(&dd->lock); \ + spin_lock_irq(&dd->lock); \ return seq_list_start(&per_prio->dispatch, *pos); \ } \ \ @@ -1035,7 +1040,7 @@ static void deadline_dispatch##prio##_stop(struct seq_file *m, void *v) \ struct request_queue *q = m->private; \ struct deadline_data *dd = q->elevator->elevator_data; \ \ - spin_unlock(&dd->lock); \ + spin_unlock_irq(&dd->lock); \ } \ \ static const struct seq_operations deadline_dispatch##prio##_seq_ops = { \ -- 2.31.1