From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-6.8 required=3.0 tests=HEADER_FROM_DIFFERENT_DOMAINS, INCLUDES_PATCH,MAILING_LIST_MULTI,SIGNED_OFF_BY,SPF_HELO_NONE,SPF_PASS autolearn=unavailable autolearn_force=no version=3.4.0 Received: from mail.kernel.org (mail.kernel.org [198.145.29.99]) by smtp.lore.kernel.org (Postfix) with ESMTP id 87777C00454 for ; Thu, 12 Dec 2019 18:24:37 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id 609D722527 for ; Thu, 12 Dec 2019 18:24:37 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1730469AbfLLSYg (ORCPT ); Thu, 12 Dec 2019 13:24:36 -0500 Received: from mga18.intel.com ([134.134.136.126]:35418 "EHLO mga18.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1730210AbfLLSYf (ORCPT ); Thu, 12 Dec 2019 13:24:35 -0500 X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga008.fm.intel.com ([10.253.24.58]) by orsmga106.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 12 Dec 2019 10:24:35 -0800 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.69,306,1571727600"; d="scan'208";a="211201169" Received: from djiang5-desk3.ch.intel.com ([143.182.136.137]) by fmsmga008.fm.intel.com with ESMTP; 12 Dec 2019 10:24:34 -0800 Subject: [PATCH RFC v2 04/14] mm: create common code from request allocation based from blk-mq code From: Dave Jiang To: dmaengine@vger.kernel.org, linux-kernel@vger.kernel.org, vkoul@kernel.org Cc: dan.j.williams@intel.com, tony.luck@intel.com, jing.lin@intel.com, ashok.raj@intel.com, sanjay.k.kumar@intel.com, megha.dey@intel.com, jacob.jun.pan@intel.com, yi.l.liu@intel.com, axboe@kernel.dk, akpm@linux-foundation.org, tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, fenghua.yu@intel.com, hpa@zytor.com Date: Thu, 12 Dec 2019 11:24:34 -0700 Message-ID: <157617507410.42350.16156693139630931510.stgit@djiang5-desk3.ch.intel.com> In-Reply-To: <157617487798.42350.4471714981643413895.stgit@djiang5-desk3.ch.intel.com> References: <157617487798.42350.4471714981643413895.stgit@djiang5-desk3.ch.intel.com> User-Agent: StGit/unknown-version MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit Sender: linux-kernel-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Move the allocation of requests from compound pages to a common function to allow usages by other callers. Since the routine has more to do with memory allocation and management, it is moved to be exported by the mempool.h and be part of mm subsystem. Signed-off-by: Dave Jiang --- block/blk-mq.c | 94 +++++++++++++---------------------------------- include/linux/mempool.h | 6 +++ mm/Makefile | 2 - mm/request_alloc.c | 95 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 69 deletions(-) create mode 100644 mm/request_alloc.c diff --git a/block/blk-mq.c b/block/blk-mq.c index 323c9cb28066..10f3fecb381a 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -26,6 +25,7 @@ #include #include #include +#include #include @@ -2015,8 +2015,6 @@ static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, unsigned int hctx_idx) { - struct page *page; - if (tags->rqs && set->ops->exit_request) { int i; @@ -2030,16 +2028,7 @@ void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, } } - while (!list_empty(&tags->page_list)) { - page = list_first_entry(&tags->page_list, struct page, lru); - list_del_init(&page->lru); - /* - * Remove kmemleak object previously allocated in - * blk_mq_alloc_rqs(). - */ - kmemleak_free(page_address(page)); - __free_pages(page, page->private); - } + request_from_pages_free(&tags->page_list); } void blk_mq_free_rq_map(struct blk_mq_tags *tags) @@ -2089,11 +2078,6 @@ struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, return tags; } -static size_t order_to_size(unsigned int order) -{ - return (size_t)PAGE_SIZE << order; -} - static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, int node) { @@ -2109,12 +2093,20 @@ static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq, return 0; } +static void blk_mq_assign_request(void *ctx, void *ptr, int idx) +{ + struct blk_mq_tags *tags = (struct blk_mq_tags *)ctx; + struct request *rq = ptr; + + tags->static_rqs[idx] = rq; +} + int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, unsigned int hctx_idx, unsigned int depth) { - unsigned int i, j, entries_per_page, max_order = 4; - size_t rq_size, left; - int node; + unsigned int i; + size_t rq_size; + int node, rc; node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx); if (node == NUMA_NO_NODE) @@ -2128,62 +2120,28 @@ int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, */ rq_size = round_up(sizeof(struct request) + set->cmd_size, cache_line_size()); - left = rq_size * depth; - - for (i = 0; i < depth; ) { - int this_order = max_order; - struct page *page; - int to_do; - void *p; - - while (this_order && left < order_to_size(this_order - 1)) - this_order--; - - do { - page = alloc_pages_node(node, - GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO, - this_order); - if (page) - break; - if (!this_order--) - break; - if (order_to_size(this_order) < rq_size) - break; - } while (1); - if (!page) - goto fail; + rc = request_from_pages_alloc((void *)tags, depth, rq_size, + &tags->page_list, 4, node, + blk_mq_assign_request); + if (rc < 0) + goto fail; - page->private = this_order; - list_add_tail(&page->lru, &tags->page_list); + for (i = 0; i < rc; i++) { + struct request *rq = tags->static_rqs[i]; - p = page_address(page); - /* - * Allow kmemleak to scan these pages as they contain pointers - * to additional allocations like via ops->init_request(). - */ - kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO); - entries_per_page = order_to_size(this_order) / rq_size; - to_do = min(entries_per_page, depth - i); - left -= to_do * rq_size; - for (j = 0; j < to_do; j++) { - struct request *rq = p; - - tags->static_rqs[i] = rq; - if (blk_mq_init_request(set, rq, hctx_idx, node)) { - tags->static_rqs[i] = NULL; - goto fail; - } - - p += rq_size; - i++; + if (blk_mq_init_request(set, rq, hctx_idx, node)) { + tags->static_rqs[i] = NULL; + rc = -ENOMEM; + goto fail; } } + return 0; fail: blk_mq_free_rqs(set, tags, hctx_idx); - return -ENOMEM; + return rc; } /* diff --git a/include/linux/mempool.h b/include/linux/mempool.h index 0c964ac107c2..5b1f6214c881 100644 --- a/include/linux/mempool.h +++ b/include/linux/mempool.h @@ -108,4 +108,10 @@ static inline mempool_t *mempool_create_page_pool(int min_nr, int order) (void *)(long)order); } +int request_from_pages_alloc(void *ctx, unsigned int depth, size_t rq_size, + struct list_head *page_list, int max_order, + int node, + void (*assign)(void *ctx, void *req, int idx)); +void request_from_pages_free(struct list_head *page_list); + #endif /* _LINUX_MEMPOOL_H */ diff --git a/mm/Makefile b/mm/Makefile index 1937cc251883..92e7d74ba7a7 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -42,7 +42,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ mm_init.o mmu_context.o percpu.o slab_common.o \ compaction.o vmacache.o \ interval_tree.o list_lru.o workingset.o \ - debug.o gup.o $(mmu-y) + debug.o gup.o request_alloc.o $(mmu-y) # Give 'page_alloc' its own module-parameter namespace page-alloc-y := page_alloc.o diff --git a/mm/request_alloc.c b/mm/request_alloc.c new file mode 100644 index 000000000000..01ebea8ccdfc --- /dev/null +++ b/mm/request_alloc.c @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Common function for struct allocation. Moved from blk-mq code + * + * Copyright (C) 2013-2014 Jens Axboe + */ +#include +#include +#include +#include +#include +#include + +void request_from_pages_free(struct list_head *page_list) +{ + struct page *page, *n; + + list_for_each_entry_safe(page, n, page_list, lru) { + list_del_init(&page->lru); + /* + * Remove kmemleak object previously allocated in + * blk_mq_alloc_rqs(). + */ + kmemleak_free(page_address(page)); + __free_pages(page, page->private); + } +} +EXPORT_SYMBOL_GPL(request_from_pages_free); + +static size_t order_to_size(unsigned int order) +{ + return (size_t)PAGE_SIZE << order; +} + +int request_from_pages_alloc(void *ctx, unsigned int depth, size_t rq_size, + struct list_head *page_list, int max_order, + int node, + void (*assign)(void *ctx, void *req, int idx)) +{ + size_t left; + unsigned int i, j, entries_per_page; + + left = rq_size * depth; + + for (i = 0; i < depth; ) { + int this_order = max_order; + struct page *page; + int to_do; + void *p; + + while (this_order && left < order_to_size(this_order - 1)) + this_order--; + + do { + page = alloc_pages_node(node, + GFP_NOIO | __GFP_NOWARN | + __GFP_NORETRY | __GFP_ZERO, + this_order); + if (page) + break; + if (!this_order--) + break; + if (order_to_size(this_order) < rq_size) + break; + } while (1); + + if (!page) + goto fail; + + page->private = this_order; + list_add_tail(&page->lru, page_list); + + p = page_address(page); + /* + * Allow kmemleak to scan these pages as they contain pointers + * to additional allocations like via ops->init_request(). + */ + kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO); + entries_per_page = order_to_size(this_order) / rq_size; + to_do = min(entries_per_page, depth - i); + left -= to_do * rq_size; + for (j = 0; j < to_do; j++) { + assign((void *)ctx, p, i); + p += rq_size; + i++; + } + } + + return i; + +fail: + request_from_pages_free(page_list); + return -ENOMEM; +} +EXPORT_SYMBOL_GPL(request_from_pages_alloc);