From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752775AbbDMChV (ORCPT ); Sun, 12 Apr 2015 22:37:21 -0400 Received: from mga01.intel.com ([192.55.52.88]:31681 "EHLO mga01.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752485AbbDMChQ (ORCPT ); Sun, 12 Apr 2015 22:37:16 -0400 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.11,567,1422950400"; d="scan'208";a="707868177" Date: Sun, 12 Apr 2015 19:36:09 -0700 (PDT) From: Vikas Shivappa X-X-Sender: vikas@vshiva-Udesk To: Marcelo Tosatti cc: Vikas Shivappa , vikas.shivappa@intel.com, x86@kernel.org, linux-kernel@vger.kernel.org, hpa@zytor.com, tglx@linutronix.de, mingo@kernel.org, tj@kernel.org, peterz@infradead.org, matt.fleming@intel.com, will.auld@intel.com, glenn.p.williamson@intel.com, kanaka.d.juvva@intel.com Subject: Re: [PATCH 3/7] x86/intel_rdt: Support cache bit mask for Intel CAT In-Reply-To: <20150409205605.GA10643@amt.cnet> Message-ID: References: <1426202167-30598-1-git-send-email-vikas.shivappa@linux.intel.com> <1426202167-30598-4-git-send-email-vikas.shivappa@linux.intel.com> <20150409205605.GA10643@amt.cnet> User-Agent: Alpine 2.10 (DEB 1266 2009-07-14) MIME-Version: 1.0 Content-Type: TEXT/PLAIN; charset=US-ASCII; format=flowed Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org On Thu, 9 Apr 2015, Marcelo Tosatti wrote: > On Thu, Mar 12, 2015 at 04:16:03PM -0700, Vikas Shivappa wrote: >> Add support for cache bit mask manipulation. The change adds a file to >> the RDT cgroup which represents the CBM(cache bit mask) for the cgroup. >> >> The RDT cgroup follows cgroup hierarchy ,mkdir and adding tasks to the >> cgroup never fails. When a child cgroup is created it inherits the >> CLOSid and the CBM from its parent. When a user changes the default >> CBM for a cgroup, a new CLOSid may be allocated if the CBM was not >> used before. If the new CBM is the one that is already used, the >> count for that CLOSid<->CBM is incremented. The changing of 'cbm' >> may fail with -ENOSPC once the kernel runs out of maximum CLOSids it >> can support. >> User can create as many cgroups as he wants but having different CBMs >> at the same time is restricted by the maximum number of CLOSids >> (multiple cgroups can have the same CBM). >> Kernel maintains a CLOSid<->cbm mapping which keeps count >> of cgroups using a CLOSid. >> >> The tasks in the CAT cgroup would get to fill the LLC cache represented >> by the cgroup's 'cbm' file. >> >> Reuse of CLOSids for cgroups with same bitmask also has following >> advantages: >> - This helps to use the scant CLOSids optimally. >> - This also implies that during context switch, write to PQR-MSR is done >> only when a task with a different bitmask is scheduled in. >> >> Signed-off-by: Vikas Shivappa >> --- >> arch/x86/include/asm/intel_rdt.h | 3 + >> arch/x86/kernel/cpu/intel_rdt.c | 205 +++++++++++++++++++++++++++++++++++++++ >> 2 files changed, 208 insertions(+) >> >> diff --git a/arch/x86/include/asm/intel_rdt.h b/arch/x86/include/asm/intel_rdt.h >> index 87af1a5..0ed28d9 100644 >> --- a/arch/x86/include/asm/intel_rdt.h >> +++ b/arch/x86/include/asm/intel_rdt.h >> @@ -4,6 +4,9 @@ >> #ifdef CONFIG_CGROUP_RDT >> >> #include >> +#define MAX_CBM_LENGTH 32 >> +#define IA32_L3_CBM_BASE 0xc90 >> +#define CBM_FROM_INDEX(x) (IA32_L3_CBM_BASE + x) >> >> struct rdt_subsys_info { >> /* Clos Bitmap to keep track of available CLOSids.*/ >> diff --git a/arch/x86/kernel/cpu/intel_rdt.c b/arch/x86/kernel/cpu/intel_rdt.c >> index 3726f41..495497a 100644 >> --- a/arch/x86/kernel/cpu/intel_rdt.c >> +++ b/arch/x86/kernel/cpu/intel_rdt.c >> @@ -33,6 +33,9 @@ static struct rdt_subsys_info rdtss_info; >> static DEFINE_MUTEX(rdt_group_mutex); >> struct intel_rdt rdt_root_group; >> >> +#define rdt_for_each_child(pos_css, parent_ir) \ >> + css_for_each_child((pos_css), &(parent_ir)->css) >> + >> static inline bool cat_supported(struct cpuinfo_x86 *c) >> { >> if (cpu_has(c, X86_FEATURE_CAT_L3)) >> @@ -83,6 +86,31 @@ static int __init rdt_late_init(void) >> late_initcall(rdt_late_init); >> >> /* >> + * Allocates a new closid from unused closids. >> + * Called with the rdt_group_mutex held. >> + */ >> + >> +static int rdt_alloc_closid(struct intel_rdt *ir) >> +{ >> + unsigned int id; >> + unsigned int maxid; >> + >> + lockdep_assert_held(&rdt_group_mutex); >> + >> + maxid = boot_cpu_data.x86_cat_closs; >> + id = find_next_zero_bit(rdtss_info.closmap, maxid, 0); >> + if (id == maxid) >> + return -ENOSPC; >> + >> + set_bit(id, rdtss_info.closmap); >> + WARN_ON(ccmap[id].cgrp_count); >> + ccmap[id].cgrp_count++; >> + ir->clos = id; >> + >> + return 0; >> +} >> + >> +/* >> * Called with the rdt_group_mutex held. >> */ >> static int rdt_free_closid(struct intel_rdt *ir) >> @@ -133,8 +161,185 @@ static void rdt_css_free(struct cgroup_subsys_state *css) >> mutex_unlock(&rdt_group_mutex); >> } >> >> +/* >> + * Tests if atleast two contiguous bits are set. >> + */ >> + >> +static inline bool cbm_is_contiguous(unsigned long var) >> +{ >> + unsigned long first_bit, zero_bit; >> + unsigned long maxcbm = MAX_CBM_LENGTH; >> + >> + if (bitmap_weight(&var, maxcbm) < 2) >> + return false; >> + >> + first_bit = find_next_bit(&var, maxcbm, 0); >> + zero_bit = find_next_zero_bit(&var, maxcbm, first_bit); >> + >> + if (find_next_bit(&var, maxcbm, zero_bit) < maxcbm) >> + return false; >> + >> + return true; >> +} >> + >> +static int cat_cbm_read(struct seq_file *m, void *v) >> +{ >> + struct intel_rdt *ir = css_rdt(seq_css(m)); >> + >> + seq_printf(m, "%08lx\n", ccmap[ir->clos].cbm); >> + return 0; >> +} >> + >> +static int validate_cbm(struct intel_rdt *ir, unsigned long cbmvalue) >> +{ >> + struct intel_rdt *par, *c; >> + struct cgroup_subsys_state *css; >> + unsigned long *cbm_tmp; >> + >> + if (!cbm_is_contiguous(cbmvalue)) { >> + pr_info("cbm should have >= 2 bits and be contiguous\n"); >> + return -EINVAL; >> + } >> + >> + par = parent_rdt(ir); >> + cbm_tmp = &ccmap[par->clos].cbm; >> + if (!bitmap_subset(&cbmvalue, cbm_tmp, MAX_CBM_LENGTH)) >> + return -EINVAL; > > Can you have different errors for the different cases? Could use -EPER > >> + rcu_read_lock(); >> + rdt_for_each_child(css, ir) { >> + c = css_rdt(css); >> + cbm_tmp = &ccmap[c->clos].cbm; >> + if (!bitmap_subset(cbm_tmp, &cbmvalue, MAX_CBM_LENGTH)) { >> + pr_info("Children's mask not a subset\n"); >> + rcu_read_unlock(); >> + return -EINVAL; >> + } >> + } >> + >> + rcu_read_unlock(); >> + return 0; >> +} >> + >> +static bool cbm_search(unsigned long cbm, int *closid) >> +{ >> + int maxid = boot_cpu_data.x86_cat_closs; >> + unsigned int i; >> + >> + for (i = 0; i < maxid; i++) >> + if (bitmap_equal(&cbm, &ccmap[i].cbm, MAX_CBM_LENGTH)) { >> + *closid = i; >> + return true; >> + } >> + >> + return false; >> +} >> + >> +static void cbmmap_dump(void) >> +{ >> + int i; >> + >> + pr_debug("CBMMAP\n"); >> + for (i = 0; i < boot_cpu_data.x86_cat_closs; i++) >> + pr_debug("cbm: 0x%x,cgrp_count: %u\n", >> + (unsigned int)ccmap[i].cbm, ccmap[i].cgrp_count); >> +} >> + >> +static void cpu_cbm_update(void *info) >> +{ >> + unsigned int closid = *((unsigned int *)info); >> + >> + wrmsrl(CBM_FROM_INDEX(closid), ccmap[closid].cbm); >> +} >> + >> +static inline void cbm_update(unsigned int closid) >> +{ >> + int pkg_id = -1; >> + int cpu; >> + >> + for_each_online_cpu(cpu) { >> + if (pkg_id == topology_physical_package_id(cpu)) >> + continue; >> + smp_call_function_single(cpu, cpu_cbm_update, &closid, 1); >> + pkg_id = topology_physical_package_id(cpu); >> + >> + > > Can use smp_call_function_many, once, more efficient. > > Can this race with CPU hotplug? BTW, on CPU hotplug, where are > the IA32_L3_MASK_n initialized for the new CPU ? > Thanks for pointing out , Will fix this . Think i was terrible when i changed the design to not use the cpuset did not change the hot cpu update , I remembered an other similar update needed.The s3 resume needs a fix to the software cache as we used the msr before. Thanks, Vikas >