[DRAFT 1/2] mm/cpuset: Exclude CDM nodes from each task's mems_allowed node mask

[Anshuman Khandual <khandual@linux.vnet.ibm.com>]

task->mems_allowed decides the final node mask of nodes from which memory
can be allocated irrespective of process or VMA based memory policy. CDM
nodes should not be used for any user space memory allocation, hence they
should not be part of any mems_allowed mask in user space to begin with.
This adds a function system_ram() which computes system RAM only nodes
and excludes all the CDM nodes on the platform. This resultant system RAM
nodemask is used instead of N_MEMORY mask during cpuset and mems_allowed
initialization. This achieves isolation of the coherent device memory
from userspace allocations.

Signed-off-by: Anshuman Khandual <khandual@linux.vnet.ibm.com>
---
This completely isolates CDM nodes from user space allocations. Hence
explicit allocation to the CDM nodes would not be possible any more.
To again enable explicit allocation capability from user space, cpuset
needs to be changed to accommodate CDM nodes into task's mems_allowed.

 include/linux/mm.h |  9 +++++++++
 kernel/cpuset.c    | 12 +++++++-----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/include/linux/mm.h b/include/linux/mm.h
index a92c8d7..f338492 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -446,6 +446,15 @@ static inline int put_page_testzero(struct page *page)
 	return page_ref_dec_and_test(page);
 }
 
+static inline nodemask_t system_ram(void)
+{
+	nodemask_t ram_nodes;
+
+	nodes_clear(ram_nodes);
+	nodes_andnot(ram_nodes, node_states[N_MEMORY], node_states[N_COHERENT_DEVICE]);
+	return ram_nodes;
+}
+
 /*
  * Try to grab a ref unless the page has a refcount of zero, return false if
  * that is the case.
diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index 29f815d..78c6fa3 100644
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -364,9 +364,11 @@ static void guarantee_online_cpus(struct cpuset *cs, struct cpumask *pmask)
  */
 static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
 {
-	while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY]))
+	nodemask_t nodes = system_ram();
+
+	while (!nodes_intersects(cs->effective_mems, nodes))
 		cs = parent_cs(cs);
-	nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]);
+	nodes_and(*pmask, cs->effective_mems, nodes);
 }
 
 /*
@@ -2301,7 +2303,7 @@ static void cpuset_hotplug_workfn(struct work_struct *work)
 
 	/* fetch the available cpus/mems and find out which changed how */
 	cpumask_copy(&new_cpus, cpu_active_mask);
-	new_mems = node_states[N_MEMORY];
+	new_mems = system_ram();
 
 	cpus_updated = !cpumask_equal(top_cpuset.effective_cpus, &new_cpus);
 	mems_updated = !nodes_equal(top_cpuset.effective_mems, new_mems);
@@ -2393,11 +2395,11 @@ static int cpuset_track_online_nodes(struct notifier_block *self,
 void __init cpuset_init_smp(void)
 {
 	cpumask_copy(top_cpuset.cpus_allowed, cpu_active_mask);
-	top_cpuset.mems_allowed = node_states[N_MEMORY];
+	top_cpuset.mems_allowed = system_ram();
 	top_cpuset.old_mems_allowed = top_cpuset.mems_allowed;
 
 	cpumask_copy(top_cpuset.effective_cpus, cpu_active_mask);
-	top_cpuset.effective_mems = node_states[N_MEMORY];
+	top_cpuset.effective_mems = system_ram();
 
 	register_hotmemory_notifier(&cpuset_track_online_nodes_nb);
 
-- 
1.8.3.1