[PATCH] blk-mq: free hctx->ctxs in queue's release handler

[PATCH] blk-mq: free hctx->ctxs in queue's release handler

[Ming Lei]

Now blk_cleanup_queue() can be called before calling
del_gendisk()[1], inside which hctx->ctxs is touched
from blk_mq_unregister_hctx(), but the variable has
been freed by blk_cleanup_queue() at that time.

So this patch moves freeing of hctx->ctxs into queue's
release handler for fixing the oops reported by Stefan.

Also ctx's kobject is embedded into each ctx pointed by
hctx->ctxs[], which shouldn't have been released so early.

[1], 6cd18e711dd8075 (block: destroy bdi before blockdev is
unregistered)

Reported-by: Stefan Seyfried <stefan.seyfried@googlemail.com>
Cc: NeilBrown <neilb@suse.de>
Cc: Christoph Hellwig <hch@lst.de>
Cc: stable@vger.kernel.org (v4.0)
Signed-off-by: Ming Lei <tom.leiming@gmail.com>
---
 block/blk-mq.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/block/blk-mq.c b/block/blk-mq.c
index e68b71b..594eea0 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -1600,6 +1600,7 @@ static int blk_mq_hctx_notify(void *data, unsigned long action,
 	return NOTIFY_OK;
 }
 
+/* hctx->ctxs will be freed in queue's release handler */
 static void blk_mq_exit_hctx(struct request_queue *q,
 		struct blk_mq_tag_set *set,
 		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
@@ -1618,7 +1619,6 @@ static void blk_mq_exit_hctx(struct request_queue *q,
 
 	blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
 	blk_free_flush_queue(hctx->fq);
-	kfree(hctx->ctxs);
 	blk_mq_free_bitmap(&hctx->ctx_map);
 }
 
@@ -1891,8 +1891,12 @@ void blk_mq_release(struct request_queue *q)
 	unsigned int i;
 
 	/* hctx kobj stays in hctx */
-	queue_for_each_hw_ctx(q, hctx, i)
+	queue_for_each_hw_ctx(q, hctx, i) {
+		if (!hctx)
+			continue;
+		kfree(hctx->ctxs);
 		kfree(hctx);
+	}
 
 	kfree(q->queue_hw_ctx);
 
-- 
1.9.1

Re: [PATCH] blk-mq: free hctx->ctxs in queue's release handler

[Ming Lei]

On Thu, Jun 4, 2015 at 9:14 PM, Ming Lei <tom.leiming@gmail.com> wrote:
> Now blk_cleanup_queue() can be called before calling
> del_gendisk()[1], inside which hctx->ctxs is touched
> from blk_mq_unregister_hctx(), but the variable has
> been freed by blk_cleanup_queue() at that time.
>
> So this patch moves freeing of hctx->ctxs into queue's
> release handler for fixing the oops reported by Stefan.
>
> Also ctx's kobject is embedded into each ctx pointed by
> hctx->ctxs[], which shouldn't have been released so early.

Hammm, the above line is wrong and should be removed, and
hctx->ctxs[] just stores the pointer of ctx.

Jens, if you need v1, please let me know.

>
> [1], 6cd18e711dd8075 (block: destroy bdi before blockdev is
> unregistered)
>
> Reported-by: Stefan Seyfried <stefan.seyfried@googlemail.com>
> Cc: NeilBrown <neilb@suse.de>
> Cc: Christoph Hellwig <hch@lst.de>
> Cc: stable@vger.kernel.org (v4.0)
> Signed-off-by: Ming Lei <tom.leiming@gmail.com>
> ---
>  block/blk-mq.c | 8 ++++++--
>  1 file changed, 6 insertions(+), 2 deletions(-)
>
> diff --git a/block/blk-mq.c b/block/blk-mq.c
> index e68b71b..594eea0 100644
> --- a/block/blk-mq.c
> +++ b/block/blk-mq.c
> @@ -1600,6 +1600,7 @@ static int blk_mq_hctx_notify(void *data, unsigned long action,
>         return NOTIFY_OK;
>  }
>
> +/* hctx->ctxs will be freed in queue's release handler */
>  static void blk_mq_exit_hctx(struct request_queue *q,
>                 struct blk_mq_tag_set *set,
>                 struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
> @@ -1618,7 +1619,6 @@ static void blk_mq_exit_hctx(struct request_queue *q,
>
>         blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
>         blk_free_flush_queue(hctx->fq);
> -       kfree(hctx->ctxs);
>         blk_mq_free_bitmap(&hctx->ctx_map);
>  }
>
> @@ -1891,8 +1891,12 @@ void blk_mq_release(struct request_queue *q)
>         unsigned int i;
>
>         /* hctx kobj stays in hctx */
> -       queue_for_each_hw_ctx(q, hctx, i)
> +       queue_for_each_hw_ctx(q, hctx, i) {
> +               if (!hctx)
> +                       continue;
> +               kfree(hctx->ctxs);
>                 kfree(hctx);
> +       }
>
>         kfree(q->queue_hw_ctx);
>
> --
> 1.9.1
>



-- 
Ming Lei

Re: [PATCH] blk-mq: free hctx->ctxs in queue's release handler

[Jens Axboe]

On 06/04/2015 07:19 AM, Ming Lei wrote:
> On Thu, Jun 4, 2015 at 9:14 PM, Ming Lei <tom.leiming@gmail.com> wrote:
>> Now blk_cleanup_queue() can be called before calling
>> del_gendisk()[1], inside which hctx->ctxs is touched
>> from blk_mq_unregister_hctx(), but the variable has
>> been freed by blk_cleanup_queue() at that time.
>>
>> So this patch moves freeing of hctx->ctxs into queue's
>> release handler for fixing the oops reported by Stefan.
>>
>> Also ctx's kobject is embedded into each ctx pointed by
>> hctx->ctxs[], which shouldn't have been released so early.
>
> Hammm, the above line is wrong and should be removed, and
> hctx->ctxs[] just stores the pointer of ctx.
>
> Jens, if you need v1, please let me know.

Yeah, lets respin it please.

-- 
Jens Axboe