From: yulei.kernel@gmail.com
To: akpm@linux-foundation.org, naoya.horiguchi@nec.com,
viro@zeniv.linux.org.uk, pbonzini@redhat.com
Cc: linux-fsdevel@vger.kernel.org, kvm@vger.kernel.org,
linux-kernel@vger.kernel.org, xiaoguangrong.eric@gmail.com,
kernellwp@gmail.com, lihaiwei.kernel@gmail.com,
Yulei Zhang <yuleixzhang@tencent.com>,
Haiwei Li <gerryhwli@tencent.com>
Subject: [PATCH 12/35] dmem: introduce mempolicy support
Date: Thu, 8 Oct 2020 15:54:02 +0800 [thread overview]
Message-ID: <1fce243b3bcd347c951a0991a6daf0645d441e4d.1602093760.git.yuleixzhang@tencent.com> (raw)
In-Reply-To: <cover.1602093760.git.yuleixzhang@tencent.com>
In-Reply-To: <cover.1602093760.git.yuleixzhang@tencent.com>
From: Yulei Zhang <yuleixzhang@tencent.com>
It adds mempolicy support for dmem to allocates memory
from mempolicy specified nodes.
Signed-off-by: Haiwei Li <gerryhwli@tencent.com>
Signed-off-by: Yulei Zhang <yuleixzhang@tencent.com>
---
arch/x86/Kconfig | 1 +
arch/x86/include/asm/pgtable.h | 7 ++++
arch/x86/include/asm/pgtable_types.h | 13 +++++-
fs/dmemfs/Kconfig | 3 ++
include/linux/pgtable.h | 7 ++++
mm/Kconfig | 3 ++
mm/dmem.c | 63 +++++++++++++++++++++++++++-
7 files changed, 94 insertions(+), 3 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 7101ac64bb20..86f3139edfc7 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -73,6 +73,7 @@ config X86
select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE
select ARCH_HAS_PMEM_API if X86_64
select ARCH_HAS_PTE_DEVMAP if X86_64
+ select ARCH_HAS_PTE_DMEM if X86_64
select ARCH_HAS_PTE_SPECIAL
select ARCH_HAS_UACCESS_FLUSHCACHE if X86_64
select ARCH_HAS_UACCESS_MCSAFE if X86_64 && X86_MCE
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index b836138ce852..ea4554a728bc 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -453,6 +453,13 @@ static inline pmd_t pmd_mkdevmap(pmd_t pmd)
return pmd_set_flags(pmd, _PAGE_DEVMAP);
}
+#ifdef CONFIG_ARCH_HAS_PTE_DMEM
+static inline pmd_t pmd_mkdmem(pmd_t pmd)
+{
+ return pmd_set_flags(pmd, _PAGE_SPECIAL | _PAGE_DMEM);
+}
+#endif
+
static inline pmd_t pmd_mkhuge(pmd_t pmd)
{
return pmd_set_flags(pmd, _PAGE_PSE);
diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index 816b31c68550..ee4cae110f5c 100644
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -23,6 +23,15 @@
#define _PAGE_BIT_SOFTW2 10 /* " */
#define _PAGE_BIT_SOFTW3 11 /* " */
#define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */
+#define _PAGE_BIT_DMEM 57 /* Flag used to indicate dmem pmd.
+ * Since _PAGE_BIT_SPECIAL is defined
+ * same as _PAGE_BIT_CPA_TEST, we can
+ * not only use _PAGE_BIT_SPECIAL, so
+ * add _PAGE_BIT_DMEM to help
+ * indicate it. Since dmem pte will
+ * never be splitting, setting
+ * _PAGE_BIT_SPECIAL for pte is enough.
+ */
#define _PAGE_BIT_SOFTW4 58 /* available for programmer */
#define _PAGE_BIT_PKEY_BIT0 59 /* Protection Keys, bit 1/4 */
#define _PAGE_BIT_PKEY_BIT1 60 /* Protection Keys, bit 2/4 */
@@ -112,9 +121,11 @@
#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
#define _PAGE_NX (_AT(pteval_t, 1) << _PAGE_BIT_NX)
#define _PAGE_DEVMAP (_AT(u64, 1) << _PAGE_BIT_DEVMAP)
+#define _PAGE_DMEM (_AT(u64, 1) << _PAGE_BIT_DMEM)
#else
#define _PAGE_NX (_AT(pteval_t, 0))
#define _PAGE_DEVMAP (_AT(pteval_t, 0))
+#define _PAGE_DMEM (_AT(pteval_t, 0))
#endif
#define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE)
@@ -128,7 +139,7 @@
#define _PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \
_PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY | \
_PAGE_SOFT_DIRTY | _PAGE_DEVMAP | _PAGE_ENC | \
- _PAGE_UFFD_WP)
+ _PAGE_UFFD_WP | _PAGE_DMEM)
#define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE)
/*
diff --git a/fs/dmemfs/Kconfig b/fs/dmemfs/Kconfig
index d2894a513de0..19ca3914da39 100644
--- a/fs/dmemfs/Kconfig
+++ b/fs/dmemfs/Kconfig
@@ -1,5 +1,8 @@
config DMEM_FS
tristate "Direct Memory filesystem support"
+ depends on DMEM
+ depends on TRANSPARENT_HUGEPAGE
+ depends on ARCH_HAS_PTE_DMEM
help
dmemfs (Direct Memory filesystem) is device memory or reserved
memory based filesystem. This kind of memory is special as it
diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
index e8cbc2e795d5..45d4c4a3e519 100644
--- a/include/linux/pgtable.h
+++ b/include/linux/pgtable.h
@@ -1129,6 +1129,13 @@ static inline int pud_trans_unstable(pud_t *pud)
#endif
}
+#ifndef CONFIG_ARCH_HAS_PTE_DMEM
+static inline pmd_t pmd_mkdmem(pmd_t pmd)
+{
+ return pmd;
+}
+#endif
+
#ifndef pmd_read_atomic
static inline pmd_t pmd_read_atomic(pmd_t *pmdp)
{
diff --git a/mm/Kconfig b/mm/Kconfig
index 8a67c8933a42..09d1b1551a44 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -795,6 +795,9 @@ config IDLE_PAGE_TRACKING
config ARCH_HAS_PTE_DEVMAP
bool
+config ARCH_HAS_PTE_DMEM
+ bool
+
config ZONE_DEVICE
bool "Device memory (pmem, HMM, etc...) hotplug support"
depends on MEMORY_HOTPLUG
diff --git a/mm/dmem.c b/mm/dmem.c
index 6992e57d5df0..2e61dbddbc62 100644
--- a/mm/dmem.c
+++ b/mm/dmem.c
@@ -822,6 +822,56 @@ dmem_alloc_pages_nodemask(int nid, nodemask_t *nodemask, unsigned int try_max,
}
EXPORT_SYMBOL(dmem_alloc_pages_nodemask);
+/* Return a nodelist indicated for current node representing a mempolicy */
+static int *policy_nodelist(struct mempolicy *policy)
+{
+ int nd = numa_node_id();
+
+ switch (policy->mode) {
+ case MPOL_PREFERRED:
+ if (!(policy->flags & MPOL_F_LOCAL))
+ nd = policy->v.preferred_node;
+ break;
+ case MPOL_BIND:
+ if (unlikely(!node_isset(nd, policy->v.nodes)))
+ nd = first_node(policy->v.nodes);
+ break;
+ default:
+ WARN_ON(1);
+ }
+ return dmem_nodelist(nd);
+}
+
+static nodemask_t *dmem_policy_nodemask(struct mempolicy *policy)
+{
+ if (unlikely(policy->mode == MPOL_BIND) &&
+ cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
+ return &policy->v.nodes;
+
+ return NULL;
+}
+
+static void
+get_mempolicy_nlist_and_nmask(struct mempolicy *pol,
+ struct vm_area_struct *vma, unsigned long addr,
+ int **nl, nodemask_t **nmask)
+{
+ if (pol->mode == MPOL_INTERLEAVE) {
+ unsigned int nid;
+
+ /*
+ * we use dpage_shift to interleave numa nodes although
+ * multiple dpages may be allocated
+ */
+ nid = interleave_nid(pol, vma, addr, dmem_pool.dpage_shift);
+ *nl = dmem_nodelist(nid);
+ *nmask = NULL;
+ } else {
+ *nl = policy_nodelist(pol);
+ *nmask = dmem_policy_nodemask(pol);
+ }
+}
+
/*
* dmem_alloc_pages_vma - Allocate pages for a VMA.
*
@@ -830,6 +880,9 @@ EXPORT_SYMBOL(dmem_alloc_pages_nodemask);
* @try_max: try to allocate @try_max dpages if possible
* @result_nr: allocated dpage number returned to the caller
*
+ * This function allocates pages from dmem pool and applies a NUMA policy
+ * associated with the VMA.
+ *
* Return the physical address of the first dpage allocated from dmem
* pool, or 0 on failure. The allocated dpage number is filled into
* @result_nr
@@ -839,13 +892,19 @@ dmem_alloc_pages_vma(struct vm_area_struct *vma, unsigned long addr,
unsigned int try_max, unsigned int *result_nr)
{
phys_addr_t phys_addr;
+ struct mempolicy *pol;
int *nl;
+ nodemask_t *nmask;
unsigned int cpuset_mems_cookie;
retry_cpuset:
- nl = dmem_nodelist(numa_node_id());
+ pol = get_vma_policy(vma, addr);
+ cpuset_mems_cookie = read_mems_allowed_begin();
+
+ get_mempolicy_nlist_and_nmask(pol, vma, addr, &nl, &nmask);
+ mpol_cond_put(pol);
- phys_addr = dmem_alloc_pages_from_nodelist(nl, NULL, try_max,
+ phys_addr = dmem_alloc_pages_from_nodelist(nl, nmask, try_max,
result_nr);
if (unlikely(!phys_addr && read_mems_allowed_retry(cpuset_mems_cookie)))
goto retry_cpuset;
--
2.28.0
next prev parent reply other threads:[~2020-10-08 7:54 UTC|newest]
Thread overview: 61+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-10-08 7:53 [PATCH 00/35] Enhance memory utilization with DMEMFS yulei.kernel
2020-10-08 7:53 ` [PATCH 01/35] fs: introduce dmemfs module yulei.kernel
2020-11-10 20:04 ` Al Viro
2020-11-11 8:53 ` yulei zhang
2020-11-11 23:09 ` Al Viro
2020-11-12 10:03 ` yulei zhang
2020-10-08 7:53 ` [PATCH 02/35] mm: support direct memory reservation yulei.kernel
2020-10-08 20:27 ` Randy Dunlap
2020-10-08 20:34 ` Randy Dunlap
2020-10-08 7:53 ` [PATCH 03/35] dmem: implement dmem memory management yulei.kernel
2020-10-08 7:53 ` [PATCH 04/35] dmem: let pat recognize dmem yulei.kernel
2020-10-13 7:27 ` Paolo Bonzini
2020-10-13 9:53 ` yulei zhang
2020-10-08 7:53 ` [PATCH 05/35] dmemfs: support mmap yulei.kernel
2020-10-08 7:53 ` [PATCH 06/35] dmemfs: support truncating inode down yulei.kernel
2020-10-08 7:53 ` [PATCH 07/35] dmem: trace core functions yulei.kernel
2020-10-08 7:53 ` [PATCH 08/35] dmem: show some statistic in debugfs yulei.kernel
2020-10-08 20:23 ` Randy Dunlap
2020-10-09 11:49 ` yulei zhang
2020-10-08 7:53 ` [PATCH 09/35] dmemfs: support remote access yulei.kernel
2020-10-08 7:54 ` [PATCH 10/35] dmemfs: introduce max_alloc_try_dpages parameter yulei.kernel
2020-10-08 7:54 ` [PATCH 11/35] mm: export mempolicy interfaces to serve dmem allocator yulei.kernel
2020-10-08 7:54 ` yulei.kernel [this message]
2020-10-08 7:54 ` [PATCH 13/35] mm, dmem: introduce PFN_DMEM and pfn_t_dmem yulei.kernel
2020-10-08 7:54 ` [PATCH 14/35] mm, dmem: dmem-pmd vs thp-pmd yulei.kernel
2020-10-08 7:54 ` [PATCH 15/35] mm: add pmd_special() check for pmd_trans_huge_lock() yulei.kernel
2020-10-08 7:54 ` [PATCH 16/35] dmemfs: introduce ->split() to dmemfs_vm_ops yulei.kernel
2020-10-08 7:54 ` [PATCH 17/35] mm, dmemfs: support unmap_page_range() for dmemfs pmd yulei.kernel
2020-10-08 7:54 ` [PATCH 18/35] mm: follow_pmd_mask() for dmem huge pmd yulei.kernel
2020-10-08 7:54 ` [PATCH 19/35] mm: gup_huge_pmd() " yulei.kernel
2020-10-08 7:54 ` [PATCH 20/35] mm: support dmem huge pmd for vmf_insert_pfn_pmd() yulei.kernel
2020-10-08 7:54 ` [PATCH 21/35] mm: support dmem huge pmd for follow_pfn() yulei.kernel
2020-10-08 7:54 ` [PATCH 22/35] kvm, x86: Distinguish dmemfs page from mmio page yulei.kernel
2020-10-09 0:58 ` Sean Christopherson
2020-10-09 10:28 ` Joao Martins
2020-10-09 11:42 ` yulei zhang
2020-10-08 7:54 ` [PATCH 23/35] kvm, x86: introduce VM_DMEM yulei.kernel
2020-10-08 7:54 ` [PATCH 24/35] dmemfs: support hugepage for dmemfs yulei.kernel
2020-10-08 7:54 ` [PATCH 25/35] mm, x86, dmem: fix estimation of reserved page for vaddr_get_pfn() yulei.kernel
2020-10-08 7:54 ` [PATCH 26/35] mm, dmem: introduce pud_special() yulei.kernel
2020-10-08 7:54 ` [PATCH 27/35] mm: add pud_special() to support dmem huge pud yulei.kernel
2020-10-08 7:54 ` [PATCH 28/35] mm, dmemfs: support huge_fault() for dmemfs yulei.kernel
2020-10-08 7:54 ` [PATCH 29/35] mm: add follow_pte_pud() yulei.kernel
2020-10-08 7:54 ` [PATCH 30/35] dmem: introduce dmem_bitmap_alloc() and dmem_bitmap_free() yulei.kernel
2020-10-08 7:54 ` [PATCH 31/35] dmem: introduce mce handler yulei.kernel
2020-10-08 7:54 ` [PATCH 32/35] mm, dmemfs: register and handle the dmem mce yulei.kernel
2020-10-08 7:54 ` [PATCH 33/35] kvm, x86: temporary disable record_steal_time for dmem yulei.kernel
2020-10-08 7:54 ` [PATCH 34/35] dmem: add dmem unit tests yulei.kernel
2020-10-08 7:54 ` [PATCH 35/35] Add documentation for dmemfs yulei.kernel
2020-10-09 1:26 ` Randy Dunlap
2020-10-08 19:01 ` [PATCH 00/35] Enhance memory utilization with DMEMFS Joao Martins
2020-10-09 11:39 ` yulei zhang
2020-10-09 11:53 ` Joao Martins
2020-10-10 8:15 ` yulei zhang
2020-10-12 10:59 ` Joao Martins
2020-10-14 22:25 ` Dan Williams
2020-10-19 13:37 ` Paolo Bonzini
2020-10-19 19:03 ` Joao Martins
2020-10-20 15:22 ` yulei zhang
2020-10-12 11:57 ` Zengtao (B)
2020-10-13 2:45 ` yulei zhang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1fce243b3bcd347c951a0991a6daf0645d441e4d.1602093760.git.yuleixzhang@tencent.com \
--to=yulei.kernel@gmail.com \
--cc=akpm@linux-foundation.org \
--cc=gerryhwli@tencent.com \
--cc=kernellwp@gmail.com \
--cc=kvm@vger.kernel.org \
--cc=lihaiwei.kernel@gmail.com \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=naoya.horiguchi@nec.com \
--cc=pbonzini@redhat.com \
--cc=viro@zeniv.linux.org.uk \
--cc=xiaoguangrong.eric@gmail.com \
--cc=yuleixzhang@tencent.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).