From: "Jérôme Glisse" <jglisse@redhat.com> To: akpm@linux-foundation.org, linux-kernel@vger.kernel.org, linux-mm@kvack.org Cc: "John Hubbard" <jhubbard@nvidia.com>, "Dan Williams" <dan.j.williams@intel.com>, "David Nellans" <dnellans@nvidia.com>, "Jérôme Glisse" <jglisse@redhat.com>, "Kirill A . Shutemov" <kirill.shutemov@linux.intel.com>, "Ross Zwisler" <ross.zwisler@linux.intel.com> Subject: [PATCH 08/15] mm/ZONE_DEVICE: special case put_page() for device private pages v2 Date: Wed, 28 Jun 2017 14:00:40 -0400 [thread overview] Message-ID: <20170628180047.5386-9-jglisse@redhat.com> (raw) In-Reply-To: <20170628180047.5386-1-jglisse@redhat.com> A ZONE_DEVICE page that reach a refcount of 1 is free ie no longer have any user. For device private pages this is important to catch and thus we need to special case put_page() for this. Changed since v1: - use static key to disable special code path in put_page() by default - uninline put_zone_device_private_page() - fix build issues with some kernel config related to header inter-dependency Signed-off-by: Jérôme Glisse <jglisse@redhat.com> Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Cc: Dan Williams <dan.j.williams@intel.com> Cc: Ross Zwisler <ross.zwisler@linux.intel.com> --- include/linux/memremap.h | 13 +++++++++++++ include/linux/mm.h | 31 ++++++++++++++++++++++--------- kernel/memremap.c | 19 ++++++++++++++++++- mm/hmm.c | 8 ++++++++ 4 files changed, 61 insertions(+), 10 deletions(-) diff --git a/include/linux/memremap.h b/include/linux/memremap.h index 45a97f7a612f..57546a07a558 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -126,6 +126,14 @@ struct dev_pagemap { void *devm_memremap_pages(struct device *dev, struct resource *res, struct percpu_ref *ref, struct vmem_altmap *altmap); struct dev_pagemap *find_dev_pagemap(resource_size_t phys); + +static inline bool is_zone_device_page(const struct page *page); + +static inline bool is_device_private_page(const struct page *page) +{ + return is_zone_device_page(page) && + page->pgmap->type == MEMORY_DEVICE_PRIVATE; +} #else static inline void *devm_memremap_pages(struct device *dev, struct resource *res, struct percpu_ref *ref, @@ -144,6 +152,11 @@ static inline struct dev_pagemap *find_dev_pagemap(resource_size_t phys) { return NULL; } + +static inline bool is_device_private_page(const struct page *page) +{ + return false; +} #endif /** diff --git a/include/linux/mm.h b/include/linux/mm.h index d53add704a7d..330a216ac315 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -23,6 +23,7 @@ #include <linux/page_ext.h> #include <linux/err.h> #include <linux/page_ref.h> +#include <linux/memremap.h> struct mempolicy; struct anon_vma; @@ -788,25 +789,25 @@ static inline bool is_zone_device_page(const struct page *page) { return page_zonenum(page) == ZONE_DEVICE; } - -static inline bool is_device_private_page(const struct page *page) -{ - /* See MEMORY_DEVICE_PRIVATE in include/linux/memory_hotplug.h */ - return ((page_zonenum(page) == ZONE_DEVICE) && - (page->pgmap->type == MEMORY_DEVICE_PRIVATE)); -} #else static inline bool is_zone_device_page(const struct page *page) { return false; } +#endif -static inline bool is_device_private_page(const struct page *page) +#ifdef CONFIG_DEVICE_PRIVATE +void put_zone_device_private_page(struct page *page); +#else +static inline void put_zone_device_private_page(struct page *page) { - return false; } #endif +static inline bool is_device_private_page(const struct page *page); + +DECLARE_STATIC_KEY_FALSE(device_private_key); + static inline void get_page(struct page *page) { page = compound_head(page); @@ -822,6 +823,18 @@ static inline void put_page(struct page *page) { page = compound_head(page); + /* + * For private device pages we need to catch refcount transition from + * 2 to 1, when refcount reach one it means the private device page is + * free and we need to inform the device driver through callback. See + * include/linux/memremap.h and HMM for details. + */ + if (static_branch_unlikely(&device_private_key) && + unlikely(is_device_private_page(page))) { + put_zone_device_private_page(page); + return; + } + if (put_page_testzero(page)) __put_page(page); } diff --git a/kernel/memremap.c b/kernel/memremap.c index cd596d4a7356..b9baa6c07918 100644 --- a/kernel/memremap.c +++ b/kernel/memremap.c @@ -11,7 +11,6 @@ * General Public License for more details. */ #include <linux/radix-tree.h> -#include <linux/memremap.h> #include <linux/device.h> #include <linux/types.h> #include <linux/pfn_t.h> @@ -464,3 +463,21 @@ struct vmem_altmap *to_vmem_altmap(unsigned long memmap_start) return pgmap ? pgmap->altmap : NULL; } #endif /* CONFIG_ZONE_DEVICE */ + + +#ifdef CONFIG_DEVICE_PRIVATE +void put_zone_device_private_page(struct page *page) +{ + int count = page_ref_dec_return(page); + + /* + * If refcount is 1 then page is freed and refcount is stable as nobody + * holds a reference on the page. + */ + if (count == 1) + page->pgmap->page_free(page, page->pgmap->data); + else if (!count) + __put_page(page); +} +EXPORT_SYMBOL(put_zone_device_private_page); +#endif /* CONFIG_DEVICE_PRIVATE */ diff --git a/mm/hmm.c b/mm/hmm.c index e7d5a363d6e5..ff9011ef51f3 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -25,9 +25,17 @@ #include <linux/sched.h> #include <linux/swapops.h> #include <linux/hugetlb.h> +#include <linux/jump_label.h> #include <linux/mmu_notifier.h> +/* + * Device private memory see HMM (Documentation/vm/hmm.txt) or hmm.h + */ +DEFINE_STATIC_KEY_FALSE(device_private_key); +EXPORT_SYMBOL(device_private_key); + + #ifdef CONFIG_HMM static const struct mmu_notifier_ops hmm_mmu_notifier_ops; -- 2.13.0
WARNING: multiple messages have this Message-ID (diff)
From: "Jérôme Glisse" <jglisse@redhat.com> To: akpm@linux-foundation.org, linux-kernel@vger.kernel.org, linux-mm@kvack.org Cc: "John Hubbard" <jhubbard@nvidia.com>, "Dan Williams" <dan.j.williams@intel.com>, "David Nellans" <dnellans@nvidia.com>, "Jérôme Glisse" <jglisse@redhat.com>, "Kirill A . Shutemov" <kirill.shutemov@linux.intel.com>, "Ross Zwisler" <ross.zwisler@linux.intel.com> Subject: [PATCH 08/15] mm/ZONE_DEVICE: special case put_page() for device private pages v2 Date: Wed, 28 Jun 2017 14:00:40 -0400 [thread overview] Message-ID: <20170628180047.5386-9-jglisse@redhat.com> (raw) In-Reply-To: <20170628180047.5386-1-jglisse@redhat.com> A ZONE_DEVICE page that reach a refcount of 1 is free ie no longer have any user. For device private pages this is important to catch and thus we need to special case put_page() for this. Changed since v1: - use static key to disable special code path in put_page() by default - uninline put_zone_device_private_page() - fix build issues with some kernel config related to header inter-dependency Signed-off-by: JA(C)rA'me Glisse <jglisse@redhat.com> Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Cc: Dan Williams <dan.j.williams@intel.com> Cc: Ross Zwisler <ross.zwisler@linux.intel.com> --- include/linux/memremap.h | 13 +++++++++++++ include/linux/mm.h | 31 ++++++++++++++++++++++--------- kernel/memremap.c | 19 ++++++++++++++++++- mm/hmm.c | 8 ++++++++ 4 files changed, 61 insertions(+), 10 deletions(-) diff --git a/include/linux/memremap.h b/include/linux/memremap.h index 45a97f7a612f..57546a07a558 100644 --- a/include/linux/memremap.h +++ b/include/linux/memremap.h @@ -126,6 +126,14 @@ struct dev_pagemap { void *devm_memremap_pages(struct device *dev, struct resource *res, struct percpu_ref *ref, struct vmem_altmap *altmap); struct dev_pagemap *find_dev_pagemap(resource_size_t phys); + +static inline bool is_zone_device_page(const struct page *page); + +static inline bool is_device_private_page(const struct page *page) +{ + return is_zone_device_page(page) && + page->pgmap->type == MEMORY_DEVICE_PRIVATE; +} #else static inline void *devm_memremap_pages(struct device *dev, struct resource *res, struct percpu_ref *ref, @@ -144,6 +152,11 @@ static inline struct dev_pagemap *find_dev_pagemap(resource_size_t phys) { return NULL; } + +static inline bool is_device_private_page(const struct page *page) +{ + return false; +} #endif /** diff --git a/include/linux/mm.h b/include/linux/mm.h index d53add704a7d..330a216ac315 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -23,6 +23,7 @@ #include <linux/page_ext.h> #include <linux/err.h> #include <linux/page_ref.h> +#include <linux/memremap.h> struct mempolicy; struct anon_vma; @@ -788,25 +789,25 @@ static inline bool is_zone_device_page(const struct page *page) { return page_zonenum(page) == ZONE_DEVICE; } - -static inline bool is_device_private_page(const struct page *page) -{ - /* See MEMORY_DEVICE_PRIVATE in include/linux/memory_hotplug.h */ - return ((page_zonenum(page) == ZONE_DEVICE) && - (page->pgmap->type == MEMORY_DEVICE_PRIVATE)); -} #else static inline bool is_zone_device_page(const struct page *page) { return false; } +#endif -static inline bool is_device_private_page(const struct page *page) +#ifdef CONFIG_DEVICE_PRIVATE +void put_zone_device_private_page(struct page *page); +#else +static inline void put_zone_device_private_page(struct page *page) { - return false; } #endif +static inline bool is_device_private_page(const struct page *page); + +DECLARE_STATIC_KEY_FALSE(device_private_key); + static inline void get_page(struct page *page) { page = compound_head(page); @@ -822,6 +823,18 @@ static inline void put_page(struct page *page) { page = compound_head(page); + /* + * For private device pages we need to catch refcount transition from + * 2 to 1, when refcount reach one it means the private device page is + * free and we need to inform the device driver through callback. See + * include/linux/memremap.h and HMM for details. + */ + if (static_branch_unlikely(&device_private_key) && + unlikely(is_device_private_page(page))) { + put_zone_device_private_page(page); + return; + } + if (put_page_testzero(page)) __put_page(page); } diff --git a/kernel/memremap.c b/kernel/memremap.c index cd596d4a7356..b9baa6c07918 100644 --- a/kernel/memremap.c +++ b/kernel/memremap.c @@ -11,7 +11,6 @@ * General Public License for more details. */ #include <linux/radix-tree.h> -#include <linux/memremap.h> #include <linux/device.h> #include <linux/types.h> #include <linux/pfn_t.h> @@ -464,3 +463,21 @@ struct vmem_altmap *to_vmem_altmap(unsigned long memmap_start) return pgmap ? pgmap->altmap : NULL; } #endif /* CONFIG_ZONE_DEVICE */ + + +#ifdef CONFIG_DEVICE_PRIVATE +void put_zone_device_private_page(struct page *page) +{ + int count = page_ref_dec_return(page); + + /* + * If refcount is 1 then page is freed and refcount is stable as nobody + * holds a reference on the page. + */ + if (count == 1) + page->pgmap->page_free(page, page->pgmap->data); + else if (!count) + __put_page(page); +} +EXPORT_SYMBOL(put_zone_device_private_page); +#endif /* CONFIG_DEVICE_PRIVATE */ diff --git a/mm/hmm.c b/mm/hmm.c index e7d5a363d6e5..ff9011ef51f3 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -25,9 +25,17 @@ #include <linux/sched.h> #include <linux/swapops.h> #include <linux/hugetlb.h> +#include <linux/jump_label.h> #include <linux/mmu_notifier.h> +/* + * Device private memory see HMM (Documentation/vm/hmm.txt) or hmm.h + */ +DEFINE_STATIC_KEY_FALSE(device_private_key); +EXPORT_SYMBOL(device_private_key); + + #ifdef CONFIG_HMM static const struct mmu_notifier_ops hmm_mmu_notifier_ops; -- 2.13.0 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to majordomo@kvack.org. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
next prev parent reply other threads:[~2017-06-28 18:03 UTC|newest] Thread overview: 52+ messages / expand[flat|nested] mbox.gz Atom feed top 2017-06-28 18:00 [PATCH 00/15] HMM (Heterogeneous Memory Management) v24 Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-06-28 18:00 ` [PATCH 01/15] hmm: heterogeneous memory management documentation v2 Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-06-28 18:00 ` [PATCH 02/15] mm/hmm: heterogeneous memory management (HMM for short) v4 Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-06-28 18:00 ` [PATCH 03/15] mm/hmm/mirror: mirror process address space on device with HMM helpers v3 Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-06-28 18:00 ` [PATCH 04/15] mm/hmm/mirror: helper to snapshot CPU page table v3 Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-06-28 18:00 ` [PATCH 05/15] mm/hmm/mirror: device page fault handler Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-06-28 18:00 ` [PATCH 06/15] mm/memory_hotplug: introduce add_pages Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-06-28 18:00 ` [PATCH 07/15] mm/ZONE_DEVICE: new type of ZONE_DEVICE for unaddressable memory v4 Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-07-01 0:16 ` kbuild test robot 2017-07-01 0:16 ` kbuild test robot 2017-06-28 18:00 ` Jérôme Glisse [this message] 2017-06-28 18:00 ` [PATCH 08/15] mm/ZONE_DEVICE: special case put_page() for device private pages v2 Jérôme Glisse 2017-06-28 18:00 ` [PATCH 09/15] mm/hmm/devmem: device memory hotplug using ZONE_DEVICE v6 Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-07-18 21:41 ` Evgeny Baskakov 2017-07-18 21:41 ` Evgeny Baskakov 2017-07-28 11:10 ` Michal Hocko 2017-07-28 11:10 ` Michal Hocko 2017-07-31 17:21 ` Jerome Glisse 2017-07-31 17:21 ` Jerome Glisse 2017-08-01 12:17 ` Michal Hocko 2017-08-01 12:17 ` Michal Hocko 2017-06-28 18:00 ` [PATCH 10/15] mm/hmm/devmem: dummy HMM device for ZONE_DEVICE memory v3 Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-06-28 18:00 ` [PATCH 11/15] mm/migrate: new migrate mode MIGRATE_SYNC_NO_COPY Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-06-28 18:00 ` [PATCH 12/15] mm/migrate: new memory migration helper for use with device memory v4 Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-06-28 18:00 ` [PATCH 13/15] mm/migrate: migrate_vma() unmap page from vma while collecting pages Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-06-28 18:00 ` [PATCH 14/15] mm/migrate: support un-addressable ZONE_DEVICE page in migration v2 Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-06-28 18:00 ` [PATCH 15/15] mm/migrate: allow migrate_vma() to alloc new page on empty entry v3 Jérôme Glisse 2017-06-28 18:00 ` Jérôme Glisse 2017-06-30 5:32 ` [PATCH 00/15] HMM (Heterogeneous Memory Management) v24 John Hubbard 2017-06-30 5:32 ` John Hubbard 2017-06-30 19:49 ` Jerome Glisse 2017-06-30 19:49 ` Jerome Glisse 2017-07-19 11:48 ` Yisheng Xie 2017-07-19 11:48 ` Yisheng Xie 2017-07-20 17:18 ` Jerome Glisse 2017-07-20 17:18 ` Jerome Glisse 2017-07-21 9:03 ` Yisheng Xie 2017-07-21 9:03 ` Yisheng Xie
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20170628180047.5386-9-jglisse@redhat.com \ --to=jglisse@redhat.com \ --cc=akpm@linux-foundation.org \ --cc=dan.j.williams@intel.com \ --cc=dnellans@nvidia.com \ --cc=jhubbard@nvidia.com \ --cc=kirill.shutemov@linux.intel.com \ --cc=linux-kernel@vger.kernel.org \ --cc=linux-mm@kvack.org \ --cc=ross.zwisler@linux.intel.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.