* [PATCH v5 01/13] x86: add socket_to_cpumask
2015-04-17 14:33 [PATCH v5 00/13] enable Cache Allocation Technology (CAT) for VMs Chao Peng
@ 2015-04-17 14:33 ` Chao Peng
2015-04-17 14:33 ` [PATCH v5 02/13] x86: improve psr scheduling code Chao Peng
` (11 subsequent siblings)
12 siblings, 0 replies; 33+ messages in thread
From: Chao Peng @ 2015-04-17 14:33 UTC (permalink / raw)
To: xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, will.auld, JBeulich, wei.liu2, dgdegra
Maintain socket_to_cpumask which contains all the HT and core siblings
in the same socket.
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
---
xen/arch/x86/smpboot.c | 15 +++++++++++++++
xen/include/asm-x86/smp.h | 8 ++++++++
2 files changed, 23 insertions(+)
diff --git a/xen/arch/x86/smpboot.c b/xen/arch/x86/smpboot.c
index 116c8f8..d236f18 100644
--- a/xen/arch/x86/smpboot.c
+++ b/xen/arch/x86/smpboot.c
@@ -59,6 +59,9 @@ DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_mask);
cpumask_t cpu_online_map __read_mostly;
EXPORT_SYMBOL(cpu_online_map);
+unsigned int nr_sockets __read_mostly;
+cpumask_t *socket_to_cpumask __read_mostly;
+
struct cpuinfo_x86 cpu_data[NR_CPUS];
u32 x86_cpu_to_apicid[NR_CPUS] __read_mostly =
@@ -301,6 +304,8 @@ static void set_cpu_sibling_map(int cpu)
}
}
}
+
+ cpumask_set_cpu(cpu, &socket_to_cpumask[cpu_to_socket(cpu)]);
}
void start_secondary(void *unused)
@@ -717,6 +722,14 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
stack_base[0] = stack_start;
+ nr_sockets = DIV_ROUND_UP(nr_cpu_ids, boot_cpu_data.x86_max_cores *
+ boot_cpu_data.x86_num_siblings);
+ ASSERT(nr_sockets > 0);
+
+ socket_to_cpumask = xzalloc_array(cpumask_t, nr_sockets);
+ if ( !socket_to_cpumask )
+ panic("No memory for socket CPU siblings map");
+
if ( !zalloc_cpumask_var(&per_cpu(cpu_sibling_mask, 0)) ||
!zalloc_cpumask_var(&per_cpu(cpu_core_mask, 0)) )
panic("No memory for boot CPU sibling/core maps");
@@ -782,6 +795,8 @@ remove_siblinginfo(int cpu)
int sibling;
struct cpuinfo_x86 *c = cpu_data;
+ cpumask_clear_cpu(cpu, &socket_to_cpumask[cpu_to_socket(cpu)]);
+
for_each_cpu ( sibling, per_cpu(cpu_core_mask, cpu) )
{
cpumask_clear_cpu(cpu, per_cpu(cpu_core_mask, sibling));
diff --git a/xen/include/asm-x86/smp.h b/xen/include/asm-x86/smp.h
index 67518cf..3ffddde 100644
--- a/xen/include/asm-x86/smp.h
+++ b/xen/include/asm-x86/smp.h
@@ -58,6 +58,14 @@ int hard_smp_processor_id(void);
void __stop_this_cpu(void);
+/*
+ * This value is considered to not change from the initial startup.
+ * Otherwise all the relevant places need to be retrofitted.
+ */
+extern unsigned int nr_sockets;
+
+/* Representing HT and core siblings in each socket */
+extern cpumask_t *socket_to_cpumask;
#endif /* !__ASSEMBLY__ */
#endif
--
1.9.1
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH v5 02/13] x86: improve psr scheduling code
2015-04-17 14:33 [PATCH v5 00/13] enable Cache Allocation Technology (CAT) for VMs Chao Peng
2015-04-17 14:33 ` [PATCH v5 01/13] x86: add socket_to_cpumask Chao Peng
@ 2015-04-17 14:33 ` Chao Peng
2015-04-20 15:42 ` Dario Faggioli
2015-04-17 14:33 ` [PATCH v5 03/13] x86: detect and initialize Intel CAT feature Chao Peng
` (10 subsequent siblings)
12 siblings, 1 reply; 33+ messages in thread
From: Chao Peng @ 2015-04-17 14:33 UTC (permalink / raw)
To: xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, will.auld, JBeulich, wei.liu2, dgdegra
Switching RMID from previous vcpu to next vcpu only needs to write
MSR_IA32_PSR_ASSOC once. Write it with the value of next vcpu is enough,
no need to write '0' first. Idle domain has RMID set to 0 and because MSR
is already updated lazily, so just switch it as it does.
Also move the initialization of per-CPU variable which used for lazy
update from context switch to CPU starting.
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
Changes in v5:
* use this_cpu() rather than per_cpu().
Changes in v4:
* Move psr_assoc_reg_read/psr_assoc_reg_write into psr_ctxt_switch_to.
* Use 0 instead of smp_processor_id() for boot cpu.
* add cpu parameter to psr_assoc_init.
Changes in v2:
* Move initialization for psr_assoc from context switch to CPU_STARTING.
---
xen/arch/x86/domain.c | 7 ++---
xen/arch/x86/psr.c | 71 +++++++++++++++++++++++++++++++++--------------
xen/include/asm-x86/psr.h | 3 +-
3 files changed, 54 insertions(+), 27 deletions(-)
diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index fcea94b..c26c732 100644
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -1444,8 +1444,6 @@ static void __context_switch(void)
{
memcpy(&p->arch.user_regs, stack_regs, CTXT_SWITCH_STACK_BYTES);
vcpu_save_fpu(p);
- if ( psr_cmt_enabled() )
- psr_assoc_rmid(0);
p->arch.ctxt_switch_from(p);
}
@@ -1470,11 +1468,10 @@ static void __context_switch(void)
}
vcpu_restore_fpu_eager(n);
n->arch.ctxt_switch_to(n);
-
- if ( psr_cmt_enabled() && n->domain->arch.psr_rmid > 0 )
- psr_assoc_rmid(n->domain->arch.psr_rmid);
}
+ psr_ctxt_switch_to(n->domain);
+
gdt = !is_pv_32on64_vcpu(n) ? per_cpu(gdt_table, cpu) :
per_cpu(compat_gdt_table, cpu);
if ( need_full_gdt(n) )
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index 344de3c..2490d22 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -22,7 +22,6 @@
struct psr_assoc {
uint64_t val;
- bool_t initialized;
};
struct psr_cmt *__read_mostly psr_cmt;
@@ -122,14 +121,6 @@ static void __init init_psr_cmt(unsigned int rmid_max)
printk(XENLOG_INFO "Cache Monitoring Technology enabled\n");
}
-static int __init init_psr(void)
-{
- if ( (opt_psr & PSR_CMT) && opt_rmid_max )
- init_psr_cmt(opt_rmid_max);
- return 0;
-}
-__initcall(init_psr);
-
/* Called with domain lock held, no psr specific lock needed */
int psr_alloc_rmid(struct domain *d)
{
@@ -175,27 +166,65 @@ void psr_free_rmid(struct domain *d)
d->arch.psr_rmid = 0;
}
-void psr_assoc_rmid(unsigned int rmid)
+static inline void psr_assoc_init(void)
{
- uint64_t val;
- uint64_t new_val;
struct psr_assoc *psra = &this_cpu(psr_assoc);
- if ( !psra->initialized )
- {
+ if ( psr_cmt_enabled() )
rdmsrl(MSR_IA32_PSR_ASSOC, psra->val);
- psra->initialized = 1;
- }
- val = psra->val;
+}
+
+static inline void psr_assoc_rmid(uint64_t *reg, unsigned int rmid)
+{
+ *reg = (*reg & ~rmid_mask) | (rmid & rmid_mask);
+}
+
+void psr_ctxt_switch_to(struct domain *d)
+{
+ struct psr_assoc *psra = &this_cpu(psr_assoc);
+ uint64_t reg = psra->val;
+
+ if ( psr_cmt_enabled() )
+ psr_assoc_rmid(®, d->arch.psr_rmid);
- new_val = (val & ~rmid_mask) | (rmid & rmid_mask);
- if ( val != new_val )
+ if ( reg != psra->val )
{
- wrmsrl(MSR_IA32_PSR_ASSOC, new_val);
- psra->val = new_val;
+ wrmsrl(MSR_IA32_PSR_ASSOC, reg);
+ psra->val = reg;
}
}
+static void psr_cpu_init(void)
+{
+ psr_assoc_init();
+}
+
+static int cpu_callback(
+ struct notifier_block *nfb, unsigned long action, void *hcpu)
+{
+ if ( action == CPU_STARTING )
+ psr_cpu_init();
+
+ return NOTIFY_DONE;
+}
+
+static struct notifier_block cpu_nfb = {
+ .notifier_call = cpu_callback
+};
+
+static int __init psr_presmp_init(void)
+{
+ if ( (opt_psr & PSR_CMT) && opt_rmid_max )
+ init_psr_cmt(opt_rmid_max);
+
+ psr_cpu_init();
+ if ( psr_cmt_enabled() )
+ register_cpu_notifier(&cpu_nfb);
+
+ return 0;
+}
+presmp_initcall(psr_presmp_init);
+
/*
* Local variables:
* mode: C
diff --git a/xen/include/asm-x86/psr.h b/xen/include/asm-x86/psr.h
index c6076e9..585350c 100644
--- a/xen/include/asm-x86/psr.h
+++ b/xen/include/asm-x86/psr.h
@@ -46,7 +46,8 @@ static inline bool_t psr_cmt_enabled(void)
int psr_alloc_rmid(struct domain *d);
void psr_free_rmid(struct domain *d);
-void psr_assoc_rmid(unsigned int rmid);
+
+void psr_ctxt_switch_to(struct domain *d);
#endif /* __ASM_PSR_H__ */
--
1.9.1
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH v5 02/13] x86: improve psr scheduling code
2015-04-17 14:33 ` [PATCH v5 02/13] x86: improve psr scheduling code Chao Peng
@ 2015-04-20 15:42 ` Dario Faggioli
0 siblings, 0 replies; 33+ messages in thread
From: Dario Faggioli @ 2015-04-20 15:42 UTC (permalink / raw)
To: Chao Peng
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, xen-devel, will.auld, JBeulich, wei.liu2, dgdegra
[-- Attachment #1.1: Type: text/plain, Size: 1020 bytes --]
On Fri, 2015-04-17 at 22:33 +0800, Chao Peng wrote:
> Switching RMID from previous vcpu to next vcpu only needs to write
> MSR_IA32_PSR_ASSOC once. Write it with the value of next vcpu is enough,
> no need to write '0' first. Idle domain has RMID set to 0 and because MSR
> is already updated lazily, so just switch it as it does.
>
> Also move the initialization of per-CPU variable which used for lazy
> update from context switch to CPU starting.
>
> Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
> Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
> ---
> --- a/xen/include/asm-x86/psr.h
> +++ b/xen/include/asm-x86/psr.h
> @@ -46,7 +46,8 @@ static inline bool_t psr_cmt_enabled(void)
>
> int psr_alloc_rmid(struct domain *d);
> void psr_free_rmid(struct domain *d);
> -void psr_assoc_rmid(unsigned int rmid);
> +
> +void psr_ctxt_switch_to(struct domain *d);
>
Why the blank line?
Anyway,
Reviewed-by: Dario Faggioli <dario.faggioli@citrix.com>
Regards,
Dario
[-- Attachment #1.2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 181 bytes --]
[-- Attachment #2: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH v5 03/13] x86: detect and initialize Intel CAT feature
2015-04-17 14:33 [PATCH v5 00/13] enable Cache Allocation Technology (CAT) for VMs Chao Peng
2015-04-17 14:33 ` [PATCH v5 01/13] x86: add socket_to_cpumask Chao Peng
2015-04-17 14:33 ` [PATCH v5 02/13] x86: improve psr scheduling code Chao Peng
@ 2015-04-17 14:33 ` Chao Peng
2015-04-20 16:13 ` Dario Faggioli
2015-04-17 14:33 ` [PATCH v5 04/13] x86: maintain COS to CBM mapping for each socket Chao Peng
` (9 subsequent siblings)
12 siblings, 1 reply; 33+ messages in thread
From: Chao Peng @ 2015-04-17 14:33 UTC (permalink / raw)
To: xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, will.auld, JBeulich, wei.liu2, dgdegra
Detect Intel Cache Allocation Technology(CAT) feature and store the
cpuid information for later use. Currently only L3 cache allocation is
supported. The L3 CAT features may vary among sockets so per-socket
feature information is stored. The initialization can happen either at
boot time or when CPU(s) is hot plugged after booting.
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
Changes in v5:
* Add cos_max boot option.
Changes in v4:
* check X86_FEATURE_CAT available before doing initialization.
Changes in v3:
* Remove num_sockets boot option instead calculate it at boot time.
* Name hardcoded CAT cpuid leaf as PSR_CPUID_LEVEL_CAT.
Changes in v2:
* socket_num => num_sockets and fix several documentaion issues.
* refactor boot line parameters parsing into standlone patch.
* set opt_num_sockets = NR_CPUS when opt_num_sockets > NR_CPUS.
* replace CPU_ONLINE with CPU_STARTING and integrate that into scheduling
improvement patch.
* reimplement get_max_socket() with cpu_to_socket();
* cbm is still uint64 as there is a path forward for supporting long masks.
---
docs/misc/xen-command-line.markdown | 15 +++++++--
xen/arch/x86/psr.c | 63 +++++++++++++++++++++++++++++++++++--
xen/include/asm-x86/cpufeature.h | 1 +
xen/include/asm-x86/psr.h | 3 ++
4 files changed, 78 insertions(+), 4 deletions(-)
diff --git a/docs/misc/xen-command-line.markdown b/docs/misc/xen-command-line.markdown
index 1dda1f0..a3deb36 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1122,9 +1122,9 @@ This option can be specified more than once (up to 8 times at present).
> `= <integer>`
### psr (Intel)
-> `= List of ( cmt:<boolean> | rmid_max:<integer> )`
+> `= List of ( cmt:<boolean> | rmid_max:<integer> | cat:<boolean> | cos_max:<integer> )`
-> Default: `psr=cmt:0,rmid_max:255`
+> Default: `psr=cmt:0,rmid_max:255,cat:0,cos_max:255`
Platform Shared Resource(PSR) Services. Intel Haswell and later server
platforms offer information about the sharing of resources.
@@ -1134,6 +1134,12 @@ Monitoring ID(RMID) is used to bind the domain to corresponding shared
resource. RMID is a hardware-provided layer of abstraction between software
and logical processors.
+To use the PSR cache allocation service for a certain domain, a capacity
+bitmasks(CBM) is used to bind the domain to corresponding shared resource.
+CBM represents cache capacity and indicates the degree of overlap and isolation
+between domains. In hypervisor a Class of Service(COS) ID is allocated for each
+unique CBM.
+
The following resources are available:
* Cache Monitoring Technology (Haswell and later). Information regarding the
@@ -1144,6 +1150,11 @@ The following resources are available:
total/local memory bandwidth. Follow the same options with Cache Monitoring
Technology.
+* Cache Alllocation Technology (Broadwell and later). Information regarding
+ the cache allocation.
+ * `cat` instructs Xen to enable/disable Cache Allocation Technology.
+ * `cos_max` indicates the max value for COS ID.
+
### reboot
> `= t[riple] | k[bd] | a[cpi] | p[ci] | P[ower] | e[fi] | n[o] [, [w]arm | [c]old]`
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index 2490d22..96456de 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -19,14 +19,25 @@
#include <asm/psr.h>
#define PSR_CMT (1<<0)
+#define PSR_CAT (1<<1)
+
+struct psr_cat_socket_info {
+ bool_t initialized;
+ bool_t enabled;
+ unsigned int cbm_len;
+ unsigned int cos_max;
+};
struct psr_assoc {
uint64_t val;
};
struct psr_cmt *__read_mostly psr_cmt;
+static struct psr_cat_socket_info *__read_mostly cat_socket_info;
+
static unsigned int __initdata opt_psr;
static unsigned int __initdata opt_rmid_max = 255;
+static unsigned int opt_cos_max = 255;
static uint64_t rmid_mask;
static DEFINE_PER_CPU(struct psr_assoc, psr_assoc);
@@ -63,10 +74,14 @@ static void __init parse_psr_param(char *s)
*val_str++ = '\0';
parse_psr_bool(s, val_str, "cmt", PSR_CMT);
+ parse_psr_bool(s, val_str, "cat", PSR_CAT);
if ( val_str && !strcmp(s, "rmid_max") )
opt_rmid_max = simple_strtoul(val_str, NULL, 0);
+ if ( val_str && !strcmp(s, "cos_max") )
+ opt_cos_max = simple_strtoul(val_str, NULL, 0);
+
s = ss + 1;
} while ( ss );
}
@@ -194,8 +209,49 @@ void psr_ctxt_switch_to(struct domain *d)
}
}
+static void cat_cpu_init(void)
+{
+ unsigned int eax, ebx, ecx, edx;
+ struct psr_cat_socket_info *info;
+ unsigned int socket;
+ unsigned int cpu = smp_processor_id();
+ const struct cpuinfo_x86 *c = cpu_data + cpu;
+
+ if ( !cpu_has(c, X86_FEATURE_CAT) )
+ return;
+
+ socket = cpu_to_socket(cpu);
+ ASSERT(socket < nr_sockets);
+
+ info = cat_socket_info + socket;
+
+ /* Avoid initializing more than one times for the same socket. */
+ if ( test_and_set_bool(info->initialized) )
+ return;
+
+ cpuid_count(PSR_CPUID_LEVEL_CAT, 0, &eax, &ebx, &ecx, &edx);
+ if ( ebx & PSR_RESOURCE_TYPE_L3 )
+ {
+ cpuid_count(PSR_CPUID_LEVEL_CAT, 1, &eax, &ebx, &ecx, &edx);
+ info->cbm_len = (eax & 0x1f) + 1;
+ info->cos_max = min(opt_cos_max, edx & 0xffff);
+
+ info->enabled = 1;
+ printk(XENLOG_INFO "CAT: enabled on socket %u, cos_max:%u, cbm_len:%u\n",
+ socket, info->cos_max, info->cbm_len);
+ }
+}
+
+static void __init init_psr_cat(void)
+{
+ cat_socket_info = xzalloc_array(struct psr_cat_socket_info, nr_sockets);
+}
+
static void psr_cpu_init(void)
{
+ if ( cat_socket_info )
+ cat_cpu_init();
+
psr_assoc_init();
}
@@ -217,9 +273,12 @@ static int __init psr_presmp_init(void)
if ( (opt_psr & PSR_CMT) && opt_rmid_max )
init_psr_cmt(opt_rmid_max);
+ if ( opt_psr & PSR_CAT )
+ init_psr_cat();
+
psr_cpu_init();
- if ( psr_cmt_enabled() )
- register_cpu_notifier(&cpu_nfb);
+ if ( psr_cmt_enabled() || cat_socket_info )
+ register_cpu_notifier(&cpu_nfb);
return 0;
}
diff --git a/xen/include/asm-x86/cpufeature.h b/xen/include/asm-x86/cpufeature.h
index 7963a3a..8c0f0a6 100644
--- a/xen/include/asm-x86/cpufeature.h
+++ b/xen/include/asm-x86/cpufeature.h
@@ -149,6 +149,7 @@
#define X86_FEATURE_CMT (7*32+12) /* Cache Monitoring Technology */
#define X86_FEATURE_NO_FPU_SEL (7*32+13) /* FPU CS/DS stored as zero */
#define X86_FEATURE_MPX (7*32+14) /* Memory Protection Extensions */
+#define X86_FEATURE_CAT (7*32+15) /* Cache Allocation Technology */
#define X86_FEATURE_RDSEED (7*32+18) /* RDSEED instruction */
#define X86_FEATURE_ADX (7*32+19) /* ADCX, ADOX instructions */
#define X86_FEATURE_SMAP (7*32+20) /* Supervisor Mode Access Prevention */
diff --git a/xen/include/asm-x86/psr.h b/xen/include/asm-x86/psr.h
index 585350c..3bc5496 100644
--- a/xen/include/asm-x86/psr.h
+++ b/xen/include/asm-x86/psr.h
@@ -18,6 +18,9 @@
#include <xen/types.h>
+/* CAT cpuid level */
+#define PSR_CPUID_LEVEL_CAT 0x10
+
/* Resource Type Enumeration */
#define PSR_RESOURCE_TYPE_L3 0x2
--
1.9.1
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH v5 03/13] x86: detect and initialize Intel CAT feature
2015-04-17 14:33 ` [PATCH v5 03/13] x86: detect and initialize Intel CAT feature Chao Peng
@ 2015-04-20 16:13 ` Dario Faggioli
2015-04-21 9:39 ` Chao Peng
0 siblings, 1 reply; 33+ messages in thread
From: Dario Faggioli @ 2015-04-20 16:13 UTC (permalink / raw)
To: Chao Peng
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, xen-devel, will.auld, JBeulich, wei.liu2, dgdegra
[-- Attachment #1.1: Type: text/plain, Size: 734 bytes --]
On Fri, 2015-04-17 at 22:33 +0800, Chao Peng wrote:
> --- a/xen/arch/x86/psr.c
> +++ b/xen/arch/x86/psr.c
> @@ -19,14 +19,25 @@
> #include <asm/psr.h>
>
> #define PSR_CMT (1<<0)
> +#define PSR_CAT (1<<1)
> +
> +struct psr_cat_socket_info {
> + bool_t initialized;
> + bool_t enabled;
> + unsigned int cbm_len;
> + unsigned int cos_max;
> +};
>
Can't we ditch 'initialized' from within the struct and have a (global)
bitmap, with one bit for each socket, expressing the same? And that also
for 'enabled'.
It's probably, at least up to a certain extent, a matter of taste (and I
personally think it will look better), but it also should produce
tighter code...
Regards,
Dario
[-- Attachment #1.2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 181 bytes --]
[-- Attachment #2: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH v5 03/13] x86: detect and initialize Intel CAT feature
2015-04-20 16:13 ` Dario Faggioli
@ 2015-04-21 9:39 ` Chao Peng
0 siblings, 0 replies; 33+ messages in thread
From: Chao Peng @ 2015-04-21 9:39 UTC (permalink / raw)
To: Dario Faggioli
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, xen-devel, will.auld, JBeulich, wei.liu2, dgdegra
On Mon, Apr 20, 2015 at 06:13:12PM +0200, Dario Faggioli wrote:
> On Fri, 2015-04-17 at 22:33 +0800, Chao Peng wrote:
>
> > --- a/xen/arch/x86/psr.c
> > +++ b/xen/arch/x86/psr.c
> > @@ -19,14 +19,25 @@
> > #include <asm/psr.h>
> >
> > #define PSR_CMT (1<<0)
> > +#define PSR_CAT (1<<1)
> > +
> > +struct psr_cat_socket_info {
> > + bool_t initialized;
> > + bool_t enabled;
> > + unsigned int cbm_len;
> > + unsigned int cos_max;
> > +};
> >
> Can't we ditch 'initialized' from within the struct and have a (global)
> bitmap, with one bit for each socket, expressing the same? And that also
> for 'enabled'.
>
> It's probably, at least up to a certain extent, a matter of taste (and I
> personally think it will look better), but it also should produce
> tighter code...
NP, thanks.
Chao
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH v5 04/13] x86: maintain COS to CBM mapping for each socket
2015-04-17 14:33 [PATCH v5 00/13] enable Cache Allocation Technology (CAT) for VMs Chao Peng
` (2 preceding siblings ...)
2015-04-17 14:33 ` [PATCH v5 03/13] x86: detect and initialize Intel CAT feature Chao Peng
@ 2015-04-17 14:33 ` Chao Peng
2015-04-17 14:33 ` [PATCH v5 05/13] x86: add COS information for each domain Chao Peng
` (8 subsequent siblings)
12 siblings, 0 replies; 33+ messages in thread
From: Chao Peng @ 2015-04-17 14:33 UTC (permalink / raw)
To: xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, will.auld, JBeulich, wei.liu2, dgdegra
For each socket, a COS to CBM mapping structure is maintained for each
COS. The mapping is indexed by COS and the value is the corresponding
CBM. Different VMs may use the same CBM, a reference count is used to
indicate if the CBM is available.
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
Changes in v5:
* rename cos_cbm_map to cos_to_cbm.
---
xen/arch/x86/psr.c | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index 96456de..11e44c4 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -21,11 +21,17 @@
#define PSR_CMT (1<<0)
#define PSR_CAT (1<<1)
+struct psr_cat_cbm {
+ unsigned int ref;
+ uint64_t cbm;
+};
+
struct psr_cat_socket_info {
bool_t initialized;
bool_t enabled;
unsigned int cbm_len;
unsigned int cos_max;
+ struct psr_cat_cbm *cos_to_cbm;
};
struct psr_assoc {
@@ -236,6 +242,14 @@ static void cat_cpu_init(void)
info->cbm_len = (eax & 0x1f) + 1;
info->cos_max = min(opt_cos_max, edx & 0xffff);
+ info->cos_to_cbm = xzalloc_array(struct psr_cat_cbm,
+ info->cos_max + 1UL);
+ if ( !info->cos_to_cbm )
+ return;
+
+ /* cos=0 is reserved as default cbm(all ones). */
+ info->cos_to_cbm[0].cbm = (1ull << info->cbm_len) - 1;
+
info->enabled = 1;
printk(XENLOG_INFO "CAT: enabled on socket %u, cos_max:%u, cbm_len:%u\n",
socket, info->cos_max, info->cbm_len);
--
1.9.1
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH v5 05/13] x86: add COS information for each domain
2015-04-17 14:33 [PATCH v5 00/13] enable Cache Allocation Technology (CAT) for VMs Chao Peng
` (3 preceding siblings ...)
2015-04-17 14:33 ` [PATCH v5 04/13] x86: maintain COS to CBM mapping for each socket Chao Peng
@ 2015-04-17 14:33 ` Chao Peng
2015-04-20 15:50 ` Andrew Cooper
2015-04-17 14:33 ` [PATCH v5 06/13] x86: expose CBM length and COS number information Chao Peng
` (7 subsequent siblings)
12 siblings, 1 reply; 33+ messages in thread
From: Chao Peng @ 2015-04-17 14:33 UTC (permalink / raw)
To: xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, will.auld, JBeulich, wei.liu2, dgdegra
In Xen's implementation, the CAT enforcement granularity is per domain.
Due to the length of CBM and the number of COS may be socket-different,
each domain has COS ID for each socket. The domain get COS=0 by default
and at runtime its COS is then allocated dynamically when user specifies
a CBM for the domain.
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
xen/arch/x86/domain.c | 6 +++++-
xen/arch/x86/psr.c | 42 ++++++++++++++++++++++++++++++++++++++++++
xen/include/asm-x86/domain.h | 5 ++++-
xen/include/asm-x86/psr.h | 3 +++
4 files changed, 54 insertions(+), 2 deletions(-)
diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index c26c732..a0b5e25 100644
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -617,6 +617,9 @@ int arch_domain_create(struct domain *d, unsigned int domcr_flags,
/* 64-bit PV guest by default. */
d->arch.is_32bit_pv = d->arch.has_32bit_shinfo = 0;
+ if ( (rc = psr_domain_init(d)) != 0 )
+ goto fail;
+
/* initialize default tsc behavior in case tools don't */
tsc_set_info(d, TSC_MODE_DEFAULT, 0UL, 0, 0);
spin_lock_init(&d->arch.vtsc_lock);
@@ -635,6 +638,7 @@ int arch_domain_create(struct domain *d, unsigned int domcr_flags,
free_perdomain_mappings(d);
if ( is_pv_domain(d) )
free_xenheap_page(d->arch.pv_domain.gdt_ldt_l1tab);
+ psr_domain_free(d);
return rc;
}
@@ -658,7 +662,7 @@ void arch_domain_destroy(struct domain *d)
free_xenheap_page(d->shared_info);
cleanup_domain_irq_mapping(d);
- psr_free_rmid(d);
+ psr_domain_free(d);
}
void arch_domain_shutdown(struct domain *d)
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index 11e44c4..592d610 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -215,6 +215,48 @@ void psr_ctxt_switch_to(struct domain *d)
}
}
+/* Called with domain lock held, no psr specific lock needed */
+static void psr_free_cos(struct domain *d)
+{
+ unsigned int socket;
+ unsigned int cos;
+
+ if( !d->arch.psr_cos_ids )
+ return;
+
+ for ( socket = 0; socket < nr_sockets; socket++ )
+ {
+ if ( !cat_socket_info[socket].enabled )
+ continue;
+
+ if ( (cos = d->arch.psr_cos_ids[socket]) == 0 )
+ continue;
+
+ cat_socket_info[socket].cos_to_cbm[cos].ref--;
+ }
+
+ xfree(d->arch.psr_cos_ids);
+ d->arch.psr_cos_ids = NULL;
+}
+
+int psr_domain_init(struct domain *d)
+{
+ if ( cat_socket_info )
+ {
+ d->arch.psr_cos_ids = xzalloc_array(unsigned int, nr_sockets);
+ if ( !d->arch.psr_cos_ids )
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+void psr_domain_free(struct domain *d)
+{
+ psr_free_rmid(d);
+ psr_free_cos(d);
+}
+
static void cat_cpu_init(void)
{
unsigned int eax, ebx, ecx, edx;
diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h
index e5102cc..324011d 100644
--- a/xen/include/asm-x86/domain.h
+++ b/xen/include/asm-x86/domain.h
@@ -333,7 +333,10 @@ struct arch_domain
struct e820entry *e820;
unsigned int nr_e820;
- unsigned int psr_rmid; /* RMID assigned to the domain for CMT */
+ /* RMID assigned to the domain for CMT */
+ unsigned int psr_rmid;
+ /* COS assigned to the domain for each socket */
+ unsigned int *psr_cos_ids;
/* Shared page for notifying that explicit PIRQ EOI is required. */
unsigned long *pirq_eoi_map;
diff --git a/xen/include/asm-x86/psr.h b/xen/include/asm-x86/psr.h
index 3bc5496..45392bf 100644
--- a/xen/include/asm-x86/psr.h
+++ b/xen/include/asm-x86/psr.h
@@ -52,6 +52,9 @@ void psr_free_rmid(struct domain *d);
void psr_ctxt_switch_to(struct domain *d);
+int psr_domain_init(struct domain *d);
+void psr_domain_free(struct domain *d);
+
#endif /* __ASM_PSR_H__ */
/*
--
1.9.1
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH v5 05/13] x86: add COS information for each domain
2015-04-17 14:33 ` [PATCH v5 05/13] x86: add COS information for each domain Chao Peng
@ 2015-04-20 15:50 ` Andrew Cooper
0 siblings, 0 replies; 33+ messages in thread
From: Andrew Cooper @ 2015-04-20 15:50 UTC (permalink / raw)
To: Chao Peng, xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, Ian.Jackson, will.auld,
JBeulich, wei.liu2, dgdegra
On 17/04/15 15:33, Chao Peng wrote:
> In Xen's implementation, the CAT enforcement granularity is per domain.
> Due to the length of CBM and the number of COS may be socket-different,
> each domain has COS ID for each socket. The domain get COS=0 by default
> and at runtime its COS is then allocated dynamically when user specifies
> a CBM for the domain.
>
> Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
> Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
> ---
> xen/arch/x86/domain.c | 6 +++++-
> xen/arch/x86/psr.c | 42 ++++++++++++++++++++++++++++++++++++++++++
> xen/include/asm-x86/domain.h | 5 ++++-
> xen/include/asm-x86/psr.h | 3 +++
> 4 files changed, 54 insertions(+), 2 deletions(-)
>
> diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
> index c26c732..a0b5e25 100644
> --- a/xen/arch/x86/domain.c
> +++ b/xen/arch/x86/domain.c
> @@ -617,6 +617,9 @@ int arch_domain_create(struct domain *d, unsigned int domcr_flags,
> /* 64-bit PV guest by default. */
> d->arch.is_32bit_pv = d->arch.has_32bit_shinfo = 0;
>
> + if ( (rc = psr_domain_init(d)) != 0 )
> + goto fail;
> +
> /* initialize default tsc behavior in case tools don't */
> tsc_set_info(d, TSC_MODE_DEFAULT, 0UL, 0, 0);
> spin_lock_init(&d->arch.vtsc_lock);
> @@ -635,6 +638,7 @@ int arch_domain_create(struct domain *d, unsigned int domcr_flags,
> free_perdomain_mappings(d);
> if ( is_pv_domain(d) )
> free_xenheap_page(d->arch.pv_domain.gdt_ldt_l1tab);
> + psr_domain_free(d);
> return rc;
> }
>
> @@ -658,7 +662,7 @@ void arch_domain_destroy(struct domain *d)
> free_xenheap_page(d->shared_info);
> cleanup_domain_irq_mapping(d);
>
> - psr_free_rmid(d);
> + psr_domain_free(d);
> }
>
> void arch_domain_shutdown(struct domain *d)
> diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
> index 11e44c4..592d610 100644
> --- a/xen/arch/x86/psr.c
> +++ b/xen/arch/x86/psr.c
> @@ -215,6 +215,48 @@ void psr_ctxt_switch_to(struct domain *d)
> }
> }
>
> +/* Called with domain lock held, no psr specific lock needed */
> +static void psr_free_cos(struct domain *d)
> +{
> + unsigned int socket;
> + unsigned int cos;
> +
> + if( !d->arch.psr_cos_ids )
> + return;
> +
> + for ( socket = 0; socket < nr_sockets; socket++ )
> + {
> + if ( !cat_socket_info[socket].enabled )
> + continue;
> +
> + if ( (cos = d->arch.psr_cos_ids[socket]) == 0 )
> + continue;
> +
> + cat_socket_info[socket].cos_to_cbm[cos].ref--;
This must be done with the spinlock held.
~Andrew
> + }
> +
> + xfree(d->arch.psr_cos_ids);
> + d->arch.psr_cos_ids = NULL;
> +}
> +
> +int psr_domain_init(struct domain *d)
> +{
> + if ( cat_socket_info )
> + {
> + d->arch.psr_cos_ids = xzalloc_array(unsigned int, nr_sockets);
> + if ( !d->arch.psr_cos_ids )
> + return -ENOMEM;
> + }
> +
> + return 0;
> +}
> +
> +void psr_domain_free(struct domain *d)
> +{
> + psr_free_rmid(d);
> + psr_free_cos(d);
> +}
> +
> static void cat_cpu_init(void)
> {
> unsigned int eax, ebx, ecx, edx;
> diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h
> index e5102cc..324011d 100644
> --- a/xen/include/asm-x86/domain.h
> +++ b/xen/include/asm-x86/domain.h
> @@ -333,7 +333,10 @@ struct arch_domain
> struct e820entry *e820;
> unsigned int nr_e820;
>
> - unsigned int psr_rmid; /* RMID assigned to the domain for CMT */
> + /* RMID assigned to the domain for CMT */
> + unsigned int psr_rmid;
> + /* COS assigned to the domain for each socket */
> + unsigned int *psr_cos_ids;
>
> /* Shared page for notifying that explicit PIRQ EOI is required. */
> unsigned long *pirq_eoi_map;
> diff --git a/xen/include/asm-x86/psr.h b/xen/include/asm-x86/psr.h
> index 3bc5496..45392bf 100644
> --- a/xen/include/asm-x86/psr.h
> +++ b/xen/include/asm-x86/psr.h
> @@ -52,6 +52,9 @@ void psr_free_rmid(struct domain *d);
>
> void psr_ctxt_switch_to(struct domain *d);
>
> +int psr_domain_init(struct domain *d);
> +void psr_domain_free(struct domain *d);
> +
> #endif /* __ASM_PSR_H__ */
>
> /*
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH v5 06/13] x86: expose CBM length and COS number information
2015-04-17 14:33 [PATCH v5 00/13] enable Cache Allocation Technology (CAT) for VMs Chao Peng
` (4 preceding siblings ...)
2015-04-17 14:33 ` [PATCH v5 05/13] x86: add COS information for each domain Chao Peng
@ 2015-04-17 14:33 ` Chao Peng
2015-04-17 14:33 ` [PATCH v5 07/13] x86: dynamically get/set CBM for a domain Chao Peng
` (6 subsequent siblings)
12 siblings, 0 replies; 33+ messages in thread
From: Chao Peng @ 2015-04-17 14:33 UTC (permalink / raw)
To: xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, will.auld, JBeulich, wei.liu2, dgdegra
General CAT information such as maximum COS and CBM length are exposed to
user space by a SYSCTL hypercall, to help user space to construct the CBM.
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
xen/arch/x86/psr.c | 31 +++++++++++++++++++++++++++++++
xen/arch/x86/sysctl.c | 18 ++++++++++++++++++
xen/include/asm-x86/psr.h | 3 +++
xen/include/public/sysctl.h | 16 ++++++++++++++++
4 files changed, 68 insertions(+)
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index 592d610..d784efb 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -215,6 +215,37 @@ void psr_ctxt_switch_to(struct domain *d)
}
}
+static int get_cat_socket_info(unsigned int socket,
+ struct psr_cat_socket_info **info)
+{
+ if ( !cat_socket_info )
+ return -ENODEV;
+
+ if ( socket >= nr_sockets )
+ return -EBADSLT;
+
+ if ( !cat_socket_info[socket].enabled )
+ return -ENOENT;
+
+ *info = cat_socket_info + socket;
+ return 0;
+}
+
+int psr_get_cat_l3_info(unsigned int socket, uint32_t *cbm_len,
+ uint32_t *cos_max)
+{
+ struct psr_cat_socket_info *info;
+ int ret = get_cat_socket_info(socket, &info);
+
+ if ( ret )
+ return ret;
+
+ *cbm_len = info->cbm_len;
+ *cos_max = info->cos_max;
+
+ return 0;
+}
+
/* Called with domain lock held, no psr specific lock needed */
static void psr_free_cos(struct domain *d)
{
diff --git a/xen/arch/x86/sysctl.c b/xen/arch/x86/sysctl.c
index 611a291..8a9e120 100644
--- a/xen/arch/x86/sysctl.c
+++ b/xen/arch/x86/sysctl.c
@@ -171,6 +171,24 @@ long arch_do_sysctl(
break;
+ case XEN_SYSCTL_psr_cat_op:
+ switch ( sysctl->u.psr_cat_op.cmd )
+ {
+ case XEN_SYSCTL_PSR_CAT_get_l3_info:
+ ret = psr_get_cat_l3_info(sysctl->u.psr_cat_op.target,
+ &sysctl->u.psr_cat_op.u.l3_info.cbm_len,
+ &sysctl->u.psr_cat_op.u.l3_info.cos_max);
+
+ if ( !ret && __copy_to_guest(u_sysctl, sysctl, 1) )
+ ret = -EFAULT;
+
+ break;
+ default:
+ ret = -EOPNOTSUPP;
+ break;
+ }
+ break;
+
default:
ret = -ENOSYS;
break;
diff --git a/xen/include/asm-x86/psr.h b/xen/include/asm-x86/psr.h
index 45392bf..3a8a406 100644
--- a/xen/include/asm-x86/psr.h
+++ b/xen/include/asm-x86/psr.h
@@ -52,6 +52,9 @@ void psr_free_rmid(struct domain *d);
void psr_ctxt_switch_to(struct domain *d);
+int psr_get_cat_l3_info(unsigned int socket, uint32_t *cbm_len,
+ uint32_t *cos_max);
+
int psr_domain_init(struct domain *d);
void psr_domain_free(struct domain *d);
diff --git a/xen/include/public/sysctl.h b/xen/include/public/sysctl.h
index 711441f..f28e460 100644
--- a/xen/include/public/sysctl.h
+++ b/xen/include/public/sysctl.h
@@ -661,6 +661,20 @@ struct xen_sysctl_psr_cmt_op {
typedef struct xen_sysctl_psr_cmt_op xen_sysctl_psr_cmt_op_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_psr_cmt_op_t);
+#define XEN_SYSCTL_PSR_CAT_get_l3_info 0
+struct xen_sysctl_psr_cat_op {
+ uint32_t cmd; /* IN: XEN_SYSCTL_PSR_CAT_* */
+ uint32_t target; /* IN: socket to be operated on */
+ union {
+ struct {
+ uint32_t cbm_len; /* OUT: CBM length */
+ uint32_t cos_max; /* OUT: Maximum COS */
+ } l3_info;
+ } u;
+};
+typedef struct xen_sysctl_psr_cat_op xen_sysctl_psr_cat_op_t;
+DEFINE_XEN_GUEST_HANDLE(xen_sysctl_psr_cat_op_t);
+
struct xen_sysctl {
uint32_t cmd;
#define XEN_SYSCTL_readconsole 1
@@ -683,6 +697,7 @@ struct xen_sysctl {
#define XEN_SYSCTL_scheduler_op 19
#define XEN_SYSCTL_coverage_op 20
#define XEN_SYSCTL_psr_cmt_op 21
+#define XEN_SYSCTL_psr_cat_op 22
uint32_t interface_version; /* XEN_SYSCTL_INTERFACE_VERSION */
union {
struct xen_sysctl_readconsole readconsole;
@@ -705,6 +720,7 @@ struct xen_sysctl {
struct xen_sysctl_scheduler_op scheduler_op;
struct xen_sysctl_coverage_op coverage_op;
struct xen_sysctl_psr_cmt_op psr_cmt_op;
+ struct xen_sysctl_psr_cat_op psr_cat_op;
uint8_t pad[128];
} u;
};
--
1.9.1
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH v5 07/13] x86: dynamically get/set CBM for a domain
2015-04-17 14:33 [PATCH v5 00/13] enable Cache Allocation Technology (CAT) for VMs Chao Peng
` (5 preceding siblings ...)
2015-04-17 14:33 ` [PATCH v5 06/13] x86: expose CBM length and COS number information Chao Peng
@ 2015-04-17 14:33 ` Chao Peng
2015-04-20 15:52 ` Andrew Cooper
2015-04-17 14:33 ` [PATCH v5 08/13] x86: add scheduling support for Intel CAT Chao Peng
` (5 subsequent siblings)
12 siblings, 1 reply; 33+ messages in thread
From: Chao Peng @ 2015-04-17 14:33 UTC (permalink / raw)
To: xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, will.auld, JBeulich, wei.liu2, dgdegra
For CAT, COS is maintained in hypervisor only while CBM is exposed to
user space directly to allow getting/setting domain's cache capacity.
For each specified CBM, hypervisor will either use a existed COS which
has the same CBM or allocate a new one if the same CBM is not found. If
the allocation fails because of no enough COS available then error is
returned. The getting/setting are always operated on a specified socket.
For multiple sockets system, the interface may be called several times.
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
---
Changes in v5:
* Add spin_lock to protect cbm_map.
---
xen/arch/x86/domctl.c | 20 ++++++
xen/arch/x86/psr.c | 139 +++++++++++++++++++++++++++++++++++++++-
xen/include/asm-x86/msr-index.h | 1 +
xen/include/asm-x86/psr.h | 2 +
xen/include/public/domctl.h | 12 ++++
5 files changed, 172 insertions(+), 2 deletions(-)
diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c
index 9450795..7ffa650 100644
--- a/xen/arch/x86/domctl.c
+++ b/xen/arch/x86/domctl.c
@@ -1334,6 +1334,26 @@ long arch_do_domctl(
}
break;
+ case XEN_DOMCTL_psr_cat_op:
+ switch ( domctl->u.psr_cat_op.cmd )
+ {
+ case XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM:
+ ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
+ domctl->u.psr_cat_op.data);
+ break;
+
+ case XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM:
+ ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
+ &domctl->u.psr_cat_op.data);
+ copyback = 1;
+ break;
+
+ default:
+ ret = -EOPNOTSUPP;
+ break;
+ }
+ break;
+
default:
ret = iommu_do_domctl(domctl, d, u_domctl);
break;
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index d784efb..2b08269 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -32,6 +32,7 @@ struct psr_cat_socket_info {
unsigned int cbm_len;
unsigned int cos_max;
struct psr_cat_cbm *cos_to_cbm;
+ spinlock_t cbm_lock;
};
struct psr_assoc {
@@ -47,6 +48,14 @@ static unsigned int opt_cos_max = 255;
static uint64_t rmid_mask;
static DEFINE_PER_CPU(struct psr_assoc, psr_assoc);
+static unsigned int get_socket_cpu(unsigned int socket)
+{
+ if ( socket < nr_sockets )
+ return cpumask_any(&socket_to_cpumask[socket]);
+
+ return nr_cpu_ids;
+}
+
static void __init parse_psr_bool(char *s, char *value, char *feature,
unsigned int mask)
{
@@ -246,24 +255,148 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t *cbm_len,
return 0;
}
+int psr_get_l3_cbm(struct domain *d, unsigned int socket, uint64_t *cbm)
+{
+ unsigned int cos;
+ struct psr_cat_socket_info *info;
+ int ret = get_cat_socket_info(socket, &info);
+
+ if ( ret )
+ return ret;
+
+ cos = d->arch.psr_cos_ids[socket];
+ *cbm = info->cos_to_cbm[cos].cbm;
+ return 0;
+}
+
+static bool_t psr_check_cbm(unsigned int cbm_len, uint64_t cbm)
+{
+ unsigned int first_bit, zero_bit;
+
+ /* Set bits should only in the range of [0, cbm_len). */
+ if ( cbm & (~0ull << cbm_len) )
+ return 0;
+
+ first_bit = find_first_bit(&cbm, cbm_len);
+ zero_bit = find_next_zero_bit(&cbm, cbm_len, first_bit);
+
+ /* Set bits should be contiguous. */
+ if ( zero_bit < cbm_len &&
+ find_next_bit(&cbm, cbm_len, zero_bit) < cbm_len )
+ return 0;
+
+ return 1;
+}
+
+struct cos_cbm_info
+{
+ unsigned int cos;
+ uint64_t cbm;
+};
+
+static void do_write_l3_cbm(void *data)
+{
+ struct cos_cbm_info *info = data;
+
+ wrmsrl(MSR_IA32_PSR_L3_MASK(info->cos), info->cbm);
+}
+
+static int write_l3_cbm(unsigned int socket, unsigned int cos, uint64_t cbm)
+{
+ struct cos_cbm_info info = { .cos = cos, .cbm = cbm };
+
+ if ( socket == cpu_to_socket(smp_processor_id()) )
+ do_write_l3_cbm(&info);
+ else
+ {
+ unsigned int cpu = get_socket_cpu(socket);
+
+ if ( cpu >= nr_cpu_ids )
+ return -EBADSLT;
+ on_selected_cpus(cpumask_of(cpu), do_write_l3_cbm, &info, 1);
+ }
+
+ return 0;
+}
+
+int psr_set_l3_cbm(struct domain *d, unsigned int socket, uint64_t cbm)
+{
+ unsigned int old_cos, cos;
+ struct psr_cat_cbm *map, *find;
+ struct psr_cat_socket_info *info;
+ int ret = get_cat_socket_info(socket, &info);
+
+ if ( ret )
+ return ret;
+
+ if ( !psr_check_cbm(info->cbm_len, cbm) )
+ return -EINVAL;
+
+ old_cos = d->arch.psr_cos_ids[socket];
+ map = info->cos_to_cbm;
+ find = NULL;
+
+ for ( cos = 0; cos <= info->cos_max; cos++ )
+ {
+ /* If still not found, then keep unused one. */
+ if ( !find && cos != 0 && map[cos].ref == 0 )
+ find = map + cos;
+ else if ( map[cos].cbm == cbm )
+ {
+ if ( unlikely(cos == old_cos) )
+ return 0;
+ find = map + cos;
+ break;
+ }
+ }
+
+ /* If old cos is referred only by the domain, then use it. */
+ if ( !find && map[old_cos].ref == 1 )
+ find = map + old_cos;
+
+ if ( !find )
+ return -EUSERS;
+
+ cos = find - map;
+ if ( find->cbm != cbm )
+ {
+ ret = write_l3_cbm(socket, cos, cbm);
+ if ( ret )
+ return ret;
+ find->cbm = cbm;
+ }
+
+ spin_lock(&info->cbm_lock);
+ find->ref++;
+ map[old_cos].ref--;
+ spin_unlock(&info->cbm_lock);
+
+ d->arch.psr_cos_ids[socket] = cos;
+ return 0;
+}
+
/* Called with domain lock held, no psr specific lock needed */
static void psr_free_cos(struct domain *d)
{
unsigned int socket;
unsigned int cos;
+ struct psr_cat_socket_info *info;
if( !d->arch.psr_cos_ids )
return;
for ( socket = 0; socket < nr_sockets; socket++ )
{
- if ( !cat_socket_info[socket].enabled )
+ info = cat_socket_info + socket;
+ if ( !info->enabled )
continue;
if ( (cos = d->arch.psr_cos_ids[socket]) == 0 )
continue;
- cat_socket_info[socket].cos_to_cbm[cos].ref--;
+ spin_lock(&info->cbm_lock);
+ info->cos_to_cbm[cos].ref--;
+ spin_unlock(&info->cbm_lock);
}
xfree(d->arch.psr_cos_ids);
@@ -323,6 +456,8 @@ static void cat_cpu_init(void)
/* cos=0 is reserved as default cbm(all ones). */
info->cos_to_cbm[0].cbm = (1ull << info->cbm_len) - 1;
+ spin_lock_init(&info->cbm_lock);
+
info->enabled = 1;
printk(XENLOG_INFO "CAT: enabled on socket %u, cos_max:%u, cbm_len:%u\n",
socket, info->cos_max, info->cbm_len);
diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h
index 83f2f70..5425f77 100644
--- a/xen/include/asm-x86/msr-index.h
+++ b/xen/include/asm-x86/msr-index.h
@@ -327,6 +327,7 @@
#define MSR_IA32_CMT_EVTSEL 0x00000c8d
#define MSR_IA32_CMT_CTR 0x00000c8e
#define MSR_IA32_PSR_ASSOC 0x00000c8f
+#define MSR_IA32_PSR_L3_MASK(n) (0x00000c90 + (n))
/* Intel Model 6 */
#define MSR_P6_PERFCTR(n) (0x000000c1 + (n))
diff --git a/xen/include/asm-x86/psr.h b/xen/include/asm-x86/psr.h
index 3a8a406..fb474bb 100644
--- a/xen/include/asm-x86/psr.h
+++ b/xen/include/asm-x86/psr.h
@@ -54,6 +54,8 @@ void psr_ctxt_switch_to(struct domain *d);
int psr_get_cat_l3_info(unsigned int socket, uint32_t *cbm_len,
uint32_t *cos_max);
+int psr_get_l3_cbm(struct domain *d, unsigned int socket, uint64_t *cbm);
+int psr_set_l3_cbm(struct domain *d, unsigned int socket, uint64_t cbm);
int psr_domain_init(struct domain *d);
void psr_domain_free(struct domain *d);
diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h
index d2e8db0..9337bb6 100644
--- a/xen/include/public/domctl.h
+++ b/xen/include/public/domctl.h
@@ -1001,6 +1001,16 @@ struct xen_domctl_psr_cmt_op {
typedef struct xen_domctl_psr_cmt_op xen_domctl_psr_cmt_op_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_psr_cmt_op_t);
+struct xen_domctl_psr_cat_op {
+#define XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM 0
+#define XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM 1
+ uint32_t cmd; /* IN: XEN_DOMCTL_PSR_CAT_OP_* */
+ uint32_t target; /* IN: socket to be operated on */
+ uint64_t data; /* IN/OUT */
+};
+typedef struct xen_domctl_psr_cat_op xen_domctl_psr_cat_op_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domctl_psr_cat_op_t);
+
struct xen_domctl {
uint32_t cmd;
#define XEN_DOMCTL_createdomain 1
@@ -1075,6 +1085,7 @@ struct xen_domctl {
#define XEN_DOMCTL_set_vcpu_msrs 73
#define XEN_DOMCTL_setvnumainfo 74
#define XEN_DOMCTL_psr_cmt_op 75
+#define XEN_DOMCTL_psr_cat_op 76
#define XEN_DOMCTL_gdbsx_guestmemio 1000
#define XEN_DOMCTL_gdbsx_pausevcpu 1001
#define XEN_DOMCTL_gdbsx_unpausevcpu 1002
@@ -1137,6 +1148,7 @@ struct xen_domctl {
struct xen_domctl_gdbsx_domstatus gdbsx_domstatus;
struct xen_domctl_vnuma vnuma;
struct xen_domctl_psr_cmt_op psr_cmt_op;
+ struct xen_domctl_psr_cat_op psr_cat_op;
uint8_t pad[128];
} u;
};
--
1.9.1
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH v5 07/13] x86: dynamically get/set CBM for a domain
2015-04-17 14:33 ` [PATCH v5 07/13] x86: dynamically get/set CBM for a domain Chao Peng
@ 2015-04-20 15:52 ` Andrew Cooper
2015-04-21 9:42 ` Chao Peng
0 siblings, 1 reply; 33+ messages in thread
From: Andrew Cooper @ 2015-04-20 15:52 UTC (permalink / raw)
To: Chao Peng, xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, Ian.Jackson, will.auld,
JBeulich, wei.liu2, dgdegra
On 17/04/15 15:33, Chao Peng wrote:
> For CAT, COS is maintained in hypervisor only while CBM is exposed to
> user space directly to allow getting/setting domain's cache capacity.
> For each specified CBM, hypervisor will either use a existed COS which
> has the same CBM or allocate a new one if the same CBM is not found. If
> the allocation fails because of no enough COS available then error is
> returned. The getting/setting are always operated on a specified socket.
> For multiple sockets system, the interface may be called several times.
>
> Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
> ---
> Changes in v5:
> * Add spin_lock to protect cbm_map.
> ---
> xen/arch/x86/domctl.c | 20 ++++++
> xen/arch/x86/psr.c | 139 +++++++++++++++++++++++++++++++++++++++-
> xen/include/asm-x86/msr-index.h | 1 +
> xen/include/asm-x86/psr.h | 2 +
> xen/include/public/domctl.h | 12 ++++
> 5 files changed, 172 insertions(+), 2 deletions(-)
>
> diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c
> index 9450795..7ffa650 100644
> --- a/xen/arch/x86/domctl.c
> +++ b/xen/arch/x86/domctl.c
> @@ -1334,6 +1334,26 @@ long arch_do_domctl(
> }
> break;
>
> + case XEN_DOMCTL_psr_cat_op:
> + switch ( domctl->u.psr_cat_op.cmd )
> + {
> + case XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM:
> + ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
> + domctl->u.psr_cat_op.data);
> + break;
> +
> + case XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM:
> + ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
> + &domctl->u.psr_cat_op.data);
> + copyback = 1;
> + break;
> +
> + default:
> + ret = -EOPNOTSUPP;
> + break;
> + }
> + break;
> +
> default:
> ret = iommu_do_domctl(domctl, d, u_domctl);
> break;
> diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
> index d784efb..2b08269 100644
> --- a/xen/arch/x86/psr.c
> +++ b/xen/arch/x86/psr.c
> @@ -32,6 +32,7 @@ struct psr_cat_socket_info {
> unsigned int cbm_len;
> unsigned int cos_max;
> struct psr_cat_cbm *cos_to_cbm;
> + spinlock_t cbm_lock;
> };
>
> struct psr_assoc {
> @@ -47,6 +48,14 @@ static unsigned int opt_cos_max = 255;
> static uint64_t rmid_mask;
> static DEFINE_PER_CPU(struct psr_assoc, psr_assoc);
>
> +static unsigned int get_socket_cpu(unsigned int socket)
> +{
> + if ( socket < nr_sockets )
> + return cpumask_any(&socket_to_cpumask[socket]);
> +
> + return nr_cpu_ids;
> +}
> +
> static void __init parse_psr_bool(char *s, char *value, char *feature,
> unsigned int mask)
> {
> @@ -246,24 +255,148 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t *cbm_len,
> return 0;
> }
>
> +int psr_get_l3_cbm(struct domain *d, unsigned int socket, uint64_t *cbm)
> +{
> + unsigned int cos;
> + struct psr_cat_socket_info *info;
> + int ret = get_cat_socket_info(socket, &info);
> +
> + if ( ret )
> + return ret;
> +
> + cos = d->arch.psr_cos_ids[socket];
> + *cbm = info->cos_to_cbm[cos].cbm;
> + return 0;
> +}
> +
> +static bool_t psr_check_cbm(unsigned int cbm_len, uint64_t cbm)
> +{
> + unsigned int first_bit, zero_bit;
> +
> + /* Set bits should only in the range of [0, cbm_len). */
> + if ( cbm & (~0ull << cbm_len) )
> + return 0;
> +
> + first_bit = find_first_bit(&cbm, cbm_len);
> + zero_bit = find_next_zero_bit(&cbm, cbm_len, first_bit);
> +
> + /* Set bits should be contiguous. */
> + if ( zero_bit < cbm_len &&
> + find_next_bit(&cbm, cbm_len, zero_bit) < cbm_len )
> + return 0;
> +
> + return 1;
> +}
> +
> +struct cos_cbm_info
> +{
> + unsigned int cos;
> + uint64_t cbm;
> +};
> +
> +static void do_write_l3_cbm(void *data)
> +{
> + struct cos_cbm_info *info = data;
> +
> + wrmsrl(MSR_IA32_PSR_L3_MASK(info->cos), info->cbm);
> +}
> +
> +static int write_l3_cbm(unsigned int socket, unsigned int cos, uint64_t cbm)
> +{
> + struct cos_cbm_info info = { .cos = cos, .cbm = cbm };
> +
> + if ( socket == cpu_to_socket(smp_processor_id()) )
> + do_write_l3_cbm(&info);
> + else
> + {
> + unsigned int cpu = get_socket_cpu(socket);
> +
> + if ( cpu >= nr_cpu_ids )
> + return -EBADSLT;
> + on_selected_cpus(cpumask_of(cpu), do_write_l3_cbm, &info, 1);
> + }
> +
> + return 0;
> +}
> +
> +int psr_set_l3_cbm(struct domain *d, unsigned int socket, uint64_t cbm)
> +{
> + unsigned int old_cos, cos;
> + struct psr_cat_cbm *map, *find;
> + struct psr_cat_socket_info *info;
> + int ret = get_cat_socket_info(socket, &info);
> +
> + if ( ret )
> + return ret;
> +
> + if ( !psr_check_cbm(info->cbm_len, cbm) )
> + return -EINVAL;
> +
> + old_cos = d->arch.psr_cos_ids[socket];
> + map = info->cos_to_cbm;
> + find = NULL;
> +
> + for ( cos = 0; cos <= info->cos_max; cos++ )
> + {
> + /* If still not found, then keep unused one. */
> + if ( !find && cos != 0 && map[cos].ref == 0 )
> + find = map + cos;
> + else if ( map[cos].cbm == cbm )
> + {
> + if ( unlikely(cos == old_cos) )
> + return 0;
> + find = map + cos;
> + break;
> + }
> + }
> +
> + /* If old cos is referred only by the domain, then use it. */
> + if ( !find && map[old_cos].ref == 1 )
> + find = map + old_cos;
> +
> + if ( !find )
> + return -EUSERS;
> +
> + cos = find - map;
> + if ( find->cbm != cbm )
> + {
> + ret = write_l3_cbm(socket, cos, cbm);
> + if ( ret )
> + return ret;
> + find->cbm = cbm;
> + }
> +
> + spin_lock(&info->cbm_lock);
> + find->ref++;
> + map[old_cos].ref--;
> + spin_unlock(&info->cbm_lock);
The spinlock must cover read accesses as well, or old_cos is liable to
be stale by this point.
It might be better to split into a rw_lock as it is read often but
modifications should be very rare.
~Andrew
> +
> + d->arch.psr_cos_ids[socket] = cos;
> + return 0;
> +}
> +
> /* Called with domain lock held, no psr specific lock needed */
> static void psr_free_cos(struct domain *d)
> {
> unsigned int socket;
> unsigned int cos;
> + struct psr_cat_socket_info *info;
>
> if( !d->arch.psr_cos_ids )
> return;
>
> for ( socket = 0; socket < nr_sockets; socket++ )
> {
> - if ( !cat_socket_info[socket].enabled )
> + info = cat_socket_info + socket;
> + if ( !info->enabled )
> continue;
>
> if ( (cos = d->arch.psr_cos_ids[socket]) == 0 )
> continue;
>
> - cat_socket_info[socket].cos_to_cbm[cos].ref--;
> + spin_lock(&info->cbm_lock);
> + info->cos_to_cbm[cos].ref--;
> + spin_unlock(&info->cbm_lock);
> }
>
> xfree(d->arch.psr_cos_ids);
> @@ -323,6 +456,8 @@ static void cat_cpu_init(void)
> /* cos=0 is reserved as default cbm(all ones). */
> info->cos_to_cbm[0].cbm = (1ull << info->cbm_len) - 1;
>
> + spin_lock_init(&info->cbm_lock);
> +
> info->enabled = 1;
> printk(XENLOG_INFO "CAT: enabled on socket %u, cos_max:%u, cbm_len:%u\n",
> socket, info->cos_max, info->cbm_len);
> diff --git a/xen/include/asm-x86/msr-index.h b/xen/include/asm-x86/msr-index.h
> index 83f2f70..5425f77 100644
> --- a/xen/include/asm-x86/msr-index.h
> +++ b/xen/include/asm-x86/msr-index.h
> @@ -327,6 +327,7 @@
> #define MSR_IA32_CMT_EVTSEL 0x00000c8d
> #define MSR_IA32_CMT_CTR 0x00000c8e
> #define MSR_IA32_PSR_ASSOC 0x00000c8f
> +#define MSR_IA32_PSR_L3_MASK(n) (0x00000c90 + (n))
>
> /* Intel Model 6 */
> #define MSR_P6_PERFCTR(n) (0x000000c1 + (n))
> diff --git a/xen/include/asm-x86/psr.h b/xen/include/asm-x86/psr.h
> index 3a8a406..fb474bb 100644
> --- a/xen/include/asm-x86/psr.h
> +++ b/xen/include/asm-x86/psr.h
> @@ -54,6 +54,8 @@ void psr_ctxt_switch_to(struct domain *d);
>
> int psr_get_cat_l3_info(unsigned int socket, uint32_t *cbm_len,
> uint32_t *cos_max);
> +int psr_get_l3_cbm(struct domain *d, unsigned int socket, uint64_t *cbm);
> +int psr_set_l3_cbm(struct domain *d, unsigned int socket, uint64_t cbm);
>
> int psr_domain_init(struct domain *d);
> void psr_domain_free(struct domain *d);
> diff --git a/xen/include/public/domctl.h b/xen/include/public/domctl.h
> index d2e8db0..9337bb6 100644
> --- a/xen/include/public/domctl.h
> +++ b/xen/include/public/domctl.h
> @@ -1001,6 +1001,16 @@ struct xen_domctl_psr_cmt_op {
> typedef struct xen_domctl_psr_cmt_op xen_domctl_psr_cmt_op_t;
> DEFINE_XEN_GUEST_HANDLE(xen_domctl_psr_cmt_op_t);
>
> +struct xen_domctl_psr_cat_op {
> +#define XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM 0
> +#define XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM 1
> + uint32_t cmd; /* IN: XEN_DOMCTL_PSR_CAT_OP_* */
> + uint32_t target; /* IN: socket to be operated on */
> + uint64_t data; /* IN/OUT */
> +};
> +typedef struct xen_domctl_psr_cat_op xen_domctl_psr_cat_op_t;
> +DEFINE_XEN_GUEST_HANDLE(xen_domctl_psr_cat_op_t);
> +
> struct xen_domctl {
> uint32_t cmd;
> #define XEN_DOMCTL_createdomain 1
> @@ -1075,6 +1085,7 @@ struct xen_domctl {
> #define XEN_DOMCTL_set_vcpu_msrs 73
> #define XEN_DOMCTL_setvnumainfo 74
> #define XEN_DOMCTL_psr_cmt_op 75
> +#define XEN_DOMCTL_psr_cat_op 76
> #define XEN_DOMCTL_gdbsx_guestmemio 1000
> #define XEN_DOMCTL_gdbsx_pausevcpu 1001
> #define XEN_DOMCTL_gdbsx_unpausevcpu 1002
> @@ -1137,6 +1148,7 @@ struct xen_domctl {
> struct xen_domctl_gdbsx_domstatus gdbsx_domstatus;
> struct xen_domctl_vnuma vnuma;
> struct xen_domctl_psr_cmt_op psr_cmt_op;
> + struct xen_domctl_psr_cat_op psr_cat_op;
> uint8_t pad[128];
> } u;
> };
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH v5 07/13] x86: dynamically get/set CBM for a domain
2015-04-20 15:52 ` Andrew Cooper
@ 2015-04-21 9:42 ` Chao Peng
0 siblings, 0 replies; 33+ messages in thread
From: Chao Peng @ 2015-04-21 9:42 UTC (permalink / raw)
To: Andrew Cooper
Cc: keir, Ian.Campbell, stefano.stabellini, Ian.Jackson, xen-devel,
will.auld, JBeulich, wei.liu2, dgdegra
On Mon, Apr 20, 2015 at 04:52:09PM +0100, Andrew Cooper wrote:
> On 17/04/15 15:33, Chao Peng wrote:
> > For CAT, COS is maintained in hypervisor only while CBM is exposed to
> > user space directly to allow getting/setting domain's cache capacity.
> > For each specified CBM, hypervisor will either use a existed COS which
> > has the same CBM or allocate a new one if the same CBM is not found. If
> > the allocation fails because of no enough COS available then error is
> > returned. The getting/setting are always operated on a specified socket.
> > For multiple sockets system, the interface may be called several times.
> >
> > Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
> > ---
> > Changes in v5:
> > * Add spin_lock to protect cbm_map.
> > ---
> > + for ( cos = 0; cos <= info->cos_max; cos++ )
> > + {
> > + /* If still not found, then keep unused one. */
> > + if ( !find && cos != 0 && map[cos].ref == 0 )
> > + find = map + cos;
> > + else if ( map[cos].cbm == cbm )
> > + {
> > + if ( unlikely(cos == old_cos) )
> > + return 0;
> > + find = map + cos;
> > + break;
> > + }
> > + }
> > +
> > + /* If old cos is referred only by the domain, then use it. */
> > + if ( !find && map[old_cos].ref == 1 )
> > + find = map + old_cos;
> > +
> > + if ( !find )
> > + return -EUSERS;
> > +
> > + cos = find - map;
> > + if ( find->cbm != cbm )
> > + {
> > + ret = write_l3_cbm(socket, cos, cbm);
> > + if ( ret )
> > + return ret;
> > + find->cbm = cbm;
> > + }
> > +
> > + spin_lock(&info->cbm_lock);
> > + find->ref++;
> > + map[old_cos].ref--;
> > + spin_unlock(&info->cbm_lock);
>
> The spinlock must cover read accesses as well, or old_cos is liable to
> be stale by this point.
You mean map[old_cos].ref and find->ref are stale, right? old_cos itself
seems not need to be protected.
>
> It might be better to split into a rw_lock as it is read often but
> modifications should be very rare.
NP, thanks.
Chao
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH v5 08/13] x86: add scheduling support for Intel CAT
2015-04-17 14:33 [PATCH v5 00/13] enable Cache Allocation Technology (CAT) for VMs Chao Peng
` (6 preceding siblings ...)
2015-04-17 14:33 ` [PATCH v5 07/13] x86: dynamically get/set CBM for a domain Chao Peng
@ 2015-04-17 14:33 ` Chao Peng
2015-04-17 14:33 ` [PATCH v5 09/13] xsm: add CAT related xsm policies Chao Peng
` (4 subsequent siblings)
12 siblings, 0 replies; 33+ messages in thread
From: Chao Peng @ 2015-04-17 14:33 UTC (permalink / raw)
To: xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, will.auld, JBeulich, wei.liu2, dgdegra
On context switch, write the the domain's Class of Service(COS) to MSR
IA32_PQR_ASSOC, to notify hardware to use the new COS.
For performance reason, the COS mask for current cpu is also cached in
the local per-CPU variable.
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
---
Changes in v5:
* Remove the need to cache socket.
Changes in v2:
* merge common scheduling changes into scheduling improvement patch.
* use readable expr for psra->cos_mask.
---
xen/arch/x86/psr.c | 24 +++++++++++++++++++++++-
1 file changed, 23 insertions(+), 1 deletion(-)
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index 2b08269..ac1c2b7 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -37,6 +37,7 @@ struct psr_cat_socket_info {
struct psr_assoc {
uint64_t val;
+ uint64_t cos_mask;
};
struct psr_cmt *__read_mostly psr_cmt;
@@ -200,7 +201,17 @@ static inline void psr_assoc_init(void)
{
struct psr_assoc *psra = &this_cpu(psr_assoc);
- if ( psr_cmt_enabled() )
+ if ( cat_socket_info )
+ {
+ struct psr_cat_socket_info *info = cat_socket_info +
+ cpu_to_socket(smp_processor_id());
+
+ if ( info->enabled )
+ psra->cos_mask = ((1ull << get_count_order(info->cos_max)) - 1)
+ << 32;
+ }
+
+ if ( psr_cmt_enabled() || psra->cos_mask )
rdmsrl(MSR_IA32_PSR_ASSOC, psra->val);
}
@@ -209,6 +220,12 @@ static inline void psr_assoc_rmid(uint64_t *reg, unsigned int rmid)
*reg = (*reg & ~rmid_mask) | (rmid & rmid_mask);
}
+static inline void psr_assoc_cos(uint64_t *reg, unsigned int cos,
+ uint64_t cos_mask)
+{
+ *reg = (*reg & ~cos_mask) | (((uint64_t)cos << 32) & cos_mask);
+}
+
void psr_ctxt_switch_to(struct domain *d)
{
struct psr_assoc *psra = &this_cpu(psr_assoc);
@@ -217,6 +234,11 @@ void psr_ctxt_switch_to(struct domain *d)
if ( psr_cmt_enabled() )
psr_assoc_rmid(®, d->arch.psr_rmid);
+ if ( psra->cos_mask )
+ psr_assoc_cos(®, d->arch.psr_cos_ids ?
+ d->arch.psr_cos_ids[cpu_to_socket(smp_processor_id())] :
+ 0, psra->cos_mask);
+
if ( reg != psra->val )
{
wrmsrl(MSR_IA32_PSR_ASSOC, reg);
--
1.9.1
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH v5 09/13] xsm: add CAT related xsm policies
2015-04-17 14:33 [PATCH v5 00/13] enable Cache Allocation Technology (CAT) for VMs Chao Peng
` (7 preceding siblings ...)
2015-04-17 14:33 ` [PATCH v5 08/13] x86: add scheduling support for Intel CAT Chao Peng
@ 2015-04-17 14:33 ` Chao Peng
2015-04-17 14:33 ` [PATCH v5 10/13] tools/libxl: minor name changes for CMT commands Chao Peng
` (3 subsequent siblings)
12 siblings, 0 replies; 33+ messages in thread
From: Chao Peng @ 2015-04-17 14:33 UTC (permalink / raw)
To: xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, will.auld, JBeulich, wei.liu2, dgdegra
Add xsm policies for Cache Allocation Technology(CAT) related hypercalls
to restrict the functions visibility to control domain only.
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
Acked-by: Daniel De Graaf <dgdegra@tycho.nsa.gov>
---
tools/flask/policy/policy/modules/xen/xen.if | 2 +-
tools/flask/policy/policy/modules/xen/xen.te | 4 +++-
xen/xsm/flask/hooks.c | 6 ++++++
xen/xsm/flask/policy/access_vectors | 4 ++++
4 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/tools/flask/policy/policy/modules/xen/xen.if b/tools/flask/policy/policy/modules/xen/xen.if
index 620d151..aa5eb72 100644
--- a/tools/flask/policy/policy/modules/xen/xen.if
+++ b/tools/flask/policy/policy/modules/xen/xen.if
@@ -51,7 +51,7 @@ define(`create_domain_common', `
getaffinity setaffinity setvcpuextstate };
allow $1 $2:domain2 { set_cpuid settsc setscheduler setclaim
set_max_evtchn set_vnumainfo get_vnumainfo cacheflush
- psr_cmt_op };
+ psr_cmt_op psr_cat_op };
allow $1 $2:security check_context;
allow $1 $2:shadow enable;
allow $1 $2:mmu { map_read map_write adjust memorymap physmap pinpage mmuext_op updatemp };
diff --git a/tools/flask/policy/policy/modules/xen/xen.te b/tools/flask/policy/policy/modules/xen/xen.te
index e555d11..6dcf953 100644
--- a/tools/flask/policy/policy/modules/xen/xen.te
+++ b/tools/flask/policy/policy/modules/xen/xen.te
@@ -67,6 +67,7 @@ allow dom0_t xen_t:xen {
allow dom0_t xen_t:xen2 {
resource_op
psr_cmt_op
+ psr_cat_op
};
allow dom0_t xen_t:mmu memorymap;
@@ -80,7 +81,8 @@ allow dom0_t dom0_t:domain {
getpodtarget setpodtarget set_misc_info set_virq_handler
};
allow dom0_t dom0_t:domain2 {
- set_cpuid gettsc settsc setscheduler set_max_evtchn set_vnumainfo get_vnumainfo psr_cmt_op
+ set_cpuid gettsc settsc setscheduler set_max_evtchn set_vnumainfo
+ get_vnumainfo psr_cmt_op psr_cat_op
};
allow dom0_t dom0_t:resource { add remove };
diff --git a/xen/xsm/flask/hooks.c b/xen/xsm/flask/hooks.c
index ab5141d..72fe9b3 100644
--- a/xen/xsm/flask/hooks.c
+++ b/xen/xsm/flask/hooks.c
@@ -729,6 +729,9 @@ static int flask_domctl(struct domain *d, int cmd)
case XEN_DOMCTL_psr_cmt_op:
return current_has_perm(d, SECCLASS_DOMAIN2, DOMAIN2__PSR_CMT_OP);
+ case XEN_DOMCTL_psr_cat_op:
+ return current_has_perm(d, SECCLASS_DOMAIN2, DOMAIN2__PSR_CAT_OP);
+
default:
printk("flask_domctl: Unknown op %d\n", cmd);
return -EPERM;
@@ -787,6 +790,9 @@ static int flask_sysctl(int cmd)
case XEN_SYSCTL_psr_cmt_op:
return avc_current_has_perm(SECINITSID_XEN, SECCLASS_XEN2,
XEN2__PSR_CMT_OP, NULL);
+ case XEN_SYSCTL_psr_cat_op:
+ return avc_current_has_perm(SECINITSID_XEN, SECCLASS_XEN2,
+ XEN2__PSR_CAT_OP, NULL);
default:
printk("flask_sysctl: Unknown op %d\n", cmd);
diff --git a/xen/xsm/flask/policy/access_vectors b/xen/xsm/flask/policy/access_vectors
index 128250e..bdf496e 100644
--- a/xen/xsm/flask/policy/access_vectors
+++ b/xen/xsm/flask/policy/access_vectors
@@ -84,6 +84,8 @@ class xen2
resource_op
# XEN_SYSCTL_psr_cmt_op
psr_cmt_op
+# XEN_SYSCTL_psr_cat_op
+ psr_cat_op
}
# Classes domain and domain2 consist of operations that a domain performs on
@@ -219,6 +221,8 @@ class domain2
get_vnumainfo
# XEN_DOMCTL_psr_cmt_op
psr_cmt_op
+# XEN_DOMCTL_psr_cat_op
+ psr_cat_op
}
# Similar to class domain, but primarily contains domctls related to HVM domains
--
1.9.1
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH v5 10/13] tools/libxl: minor name changes for CMT commands
2015-04-17 14:33 [PATCH v5 00/13] enable Cache Allocation Technology (CAT) for VMs Chao Peng
` (8 preceding siblings ...)
2015-04-17 14:33 ` [PATCH v5 09/13] xsm: add CAT related xsm policies Chao Peng
@ 2015-04-17 14:33 ` Chao Peng
2015-04-20 16:07 ` Dario Faggioli
2015-04-17 14:33 ` [PATCH v5 11/13] tools/libxl: add command to show CMT hardware info Chao Peng
` (2 subsequent siblings)
12 siblings, 1 reply; 33+ messages in thread
From: Chao Peng @ 2015-04-17 14:33 UTC (permalink / raw)
To: xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, will.auld, JBeulich, wei.liu2, dgdegra
Use "-" instead of "_" for monitor types.
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
---
tools/libxl/xl_cmdimpl.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 394b55d..c666d84 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -8220,11 +8220,11 @@ int main_psr_cmt_show(int argc, char **argv)
/* No options */
}
- if (!strcmp(argv[optind], "cache_occupancy"))
+ if (!strcmp(argv[optind], "cache-occupancy"))
type = LIBXL_PSR_CMT_TYPE_CACHE_OCCUPANCY;
- else if (!strcmp(argv[optind], "total_mem_bandwidth"))
+ else if (!strcmp(argv[optind], "total-mem-bandwidth"))
type = LIBXL_PSR_CMT_TYPE_TOTAL_MEM_COUNT;
- else if (!strcmp(argv[optind], "local_mem_bandwidth"))
+ else if (!strcmp(argv[optind], "local-mem-bandwidth"))
type = LIBXL_PSR_CMT_TYPE_LOCAL_MEM_COUNT;
else {
help("psr-cmt-show");
--
1.9.1
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH v5 10/13] tools/libxl: minor name changes for CMT commands
2015-04-17 14:33 ` [PATCH v5 10/13] tools/libxl: minor name changes for CMT commands Chao Peng
@ 2015-04-20 16:07 ` Dario Faggioli
2015-04-21 13:56 ` Ian Campbell
0 siblings, 1 reply; 33+ messages in thread
From: Dario Faggioli @ 2015-04-20 16:07 UTC (permalink / raw)
To: Chao Peng
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, xen-devel, will.auld, JBeulich, wei.liu2, dgdegra
[-- Attachment #1.1: Type: text/plain, Size: 282 bytes --]
On Fri, 2015-04-17 at 22:33 +0800, Chao Peng wrote:
> Use "-" instead of "_" for monitor types.
>
This easier to type, so I like it. :-)
> Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
>
Reviewed-by: Dario Faggioli <dario.faggioli@citrix.com>
Regards,
Dario
[-- Attachment #1.2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 181 bytes --]
[-- Attachment #2: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 33+ messages in thread
* Re: [PATCH v5 10/13] tools/libxl: minor name changes for CMT commands
2015-04-20 16:07 ` Dario Faggioli
@ 2015-04-21 13:56 ` Ian Campbell
0 siblings, 0 replies; 33+ messages in thread
From: Ian Campbell @ 2015-04-21 13:56 UTC (permalink / raw)
To: Dario Faggioli
Cc: wei.liu2, keir, stefano.stabellini, andrew.cooper3, Ian.Jackson,
xen-devel, will.auld, JBeulich, Chao Peng, dgdegra
On Mon, 2015-04-20 at 18:07 +0200, Dario Faggioli wrote:
> On Fri, 2015-04-17 at 22:33 +0800, Chao Peng wrote:
> > Use "-" instead of "_" for monitor types.
> >
> This easier to type, so I like it. :-)
>
> > Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
> >
> Reviewed-by: Dario Faggioli <dario.faggioli@citrix.com>
Acked-by: Ian Campbell <ian.campbell@citrix.com>
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH v5 11/13] tools/libxl: add command to show CMT hardware info
2015-04-17 14:33 [PATCH v5 00/13] enable Cache Allocation Technology (CAT) for VMs Chao Peng
` (9 preceding siblings ...)
2015-04-17 14:33 ` [PATCH v5 10/13] tools/libxl: minor name changes for CMT commands Chao Peng
@ 2015-04-17 14:33 ` Chao Peng
2015-04-21 0:37 ` Dario Faggioli
2015-04-17 14:33 ` [PATCH v5 12/13] tools: add tools support for Intel CAT Chao Peng
2015-04-17 14:33 ` [PATCH v5 13/13] docs: add xl-psr.markdown Chao Peng
12 siblings, 1 reply; 33+ messages in thread
From: Chao Peng @ 2015-04-17 14:33 UTC (permalink / raw)
To: xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, will.auld, JBeulich, wei.liu2, dgdegra
Add dedicated one to show hardware information.
[root@vmm-psr]xl psr-cmt-hwinfo
Cache Monitoring Technology (CMT):
Enabled : 1
Total RMID : 63
Supported monitor types:
cache-occupancy
total-mem-bandwidth
local-mem-bandwidth
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
---
docs/man/xl.pod.1 | 4 ++++
tools/libxl/xl.h | 1 +
tools/libxl/xl_cmdimpl.c | 35 +++++++++++++++++++++++++++++++++++
tools/libxl/xl_cmdtable.c | 5 +++++
4 files changed, 45 insertions(+)
diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index 16783c8..640788f 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1495,6 +1495,10 @@ for any of these monitoring types.
=over 4
+=item B<psr-cmt-hwinfo>
+
+Show CMT hardware information.
+
=item B<psr-cmt-attach> [I<domain-id>]
attach: Attach the platform shared resource monitoring service to a domain.
diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h
index 5bc138c..1c526dc 100644
--- a/tools/libxl/xl.h
+++ b/tools/libxl/xl.h
@@ -113,6 +113,7 @@ int main_remus(int argc, char **argv);
#endif
int main_devd(int argc, char **argv);
#ifdef LIBXL_HAVE_PSR_CMT
+int main_psr_cmt_hwinfo(int argc, char **argv);
int main_psr_cmt_attach(int argc, char **argv);
int main_psr_cmt_detach(int argc, char **argv);
int main_psr_cmt_show(int argc, char **argv);
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index c666d84..4ab5e8b 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -8014,6 +8014,36 @@ out:
}
#ifdef LIBXL_HAVE_PSR_CMT
+static int psr_cmt_hwinfo(void)
+{
+ int rc;
+ int enabled;
+ uint32_t total_rmid;
+
+ printf("Cache Monitoring Technology (CMT):\n");
+
+ enabled = libxl_psr_cmt_enabled(ctx);
+ printf("%-16s: %s\n", "Enabled", enabled ? "1" : "0");
+ if (!enabled)
+ return 0;
+
+ rc = libxl_psr_cmt_get_total_rmid(ctx, &total_rmid);
+ if (rc) {
+ fprintf(stderr, "Failed to get max RMID value\n");
+ return rc;
+ }
+ printf("%-16s: %u\n", "Total RMID", total_rmid);
+
+ printf("Supported monitor types:\n");
+ if (libxl_psr_cmt_type_supported(ctx, LIBXL_PSR_CMT_TYPE_CACHE_OCCUPANCY))
+ printf("cache-occupancy\n");
+ if (libxl_psr_cmt_type_supported(ctx, LIBXL_PSR_CMT_TYPE_TOTAL_MEM_COUNT))
+ printf("total-mem-bandwidth\n");
+ if (libxl_psr_cmt_type_supported(ctx, LIBXL_PSR_CMT_TYPE_LOCAL_MEM_COUNT))
+ printf("local-mem-bandwidth\n");
+
+ return rc;
+}
#define MBM_SAMPLE_RETRY_MAX 4
static int psr_cmt_get_mem_bandwidth(uint32_t domid,
@@ -8180,6 +8210,11 @@ static int psr_cmt_show(libxl_psr_cmt_type type, uint32_t domid)
return 0;
}
+int main_psr_cmt_hwinfo(int argc, char **argv)
+{
+ return psr_cmt_hwinfo();
+}
+
int main_psr_cmt_attach(int argc, char **argv)
{
uint32_t domid;
diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c
index 9284887..dc25d1f 100644
--- a/tools/libxl/xl_cmdtable.c
+++ b/tools/libxl/xl_cmdtable.c
@@ -524,6 +524,11 @@ struct cmd_spec cmd_table[] = {
"-F Run in the foreground",
},
#ifdef LIBXL_HAVE_PSR_CMT
+ { "psr-cmt-hwinfo",
+ &main_psr_cmt_hwinfo, 0, 1,
+ "Show hardware information for Cache Monitoring Technology",
+ "",
+ },
{ "psr-cmt-attach",
&main_psr_cmt_attach, 0, 1,
"Attach Cache Monitoring Technology service to a domain",
--
1.9.1
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH v5 11/13] tools/libxl: add command to show CMT hardware info
2015-04-17 14:33 ` [PATCH v5 11/13] tools/libxl: add command to show CMT hardware info Chao Peng
@ 2015-04-21 0:37 ` Dario Faggioli
2015-04-21 9:42 ` Chao Peng
2015-04-21 13:57 ` Ian Campbell
0 siblings, 2 replies; 33+ messages in thread
From: Dario Faggioli @ 2015-04-21 0:37 UTC (permalink / raw)
To: Chao Peng
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, xen-devel, will.auld, JBeulich, wei.liu2, dgdegra
[-- Attachment #1.1: Type: text/plain, Size: 946 bytes --]
On Fri, 2015-04-17 at 22:33 +0800, Chao Peng wrote:
> Add dedicated one to show hardware information.
>
> [root@vmm-psr]xl psr-cmt-hwinfo
> Cache Monitoring Technology (CMT):
> Enabled : 1
> Total RMID : 63
> Supported monitor types:
> cache-occupancy
> total-mem-bandwidth
> local-mem-bandwidth
>
Nice.
> Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
> --- a/tools/libxl/xl_cmdimpl.c
> +++ b/tools/libxl/xl_cmdimpl.c
> @@ -8014,6 +8014,36 @@ out:
> }
>
> #ifdef LIBXL_HAVE_PSR_CMT
> +static int psr_cmt_hwinfo(void)
> +{
> + int rc;
> + int enabled;
> + uint32_t total_rmid;
> +
>
I think you should still have something like:
SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cmt-hwinfo", 0) {
/* No options */
}
Or `xl psr-cmt-hwinfo -h' wouldn't work, would it?
With that done:
Reviewed-by: Dario Faggioli <dario.faggioli@citrix.com>
Regards,
Dario
[-- Attachment #1.2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 181 bytes --]
[-- Attachment #2: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH v5 11/13] tools/libxl: add command to show CMT hardware info
2015-04-21 0:37 ` Dario Faggioli
@ 2015-04-21 9:42 ` Chao Peng
2015-04-21 13:57 ` Ian Campbell
1 sibling, 0 replies; 33+ messages in thread
From: Chao Peng @ 2015-04-21 9:42 UTC (permalink / raw)
To: Dario Faggioli
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, xen-devel, will.auld, JBeulich, wei.liu2, dgdegra
On Tue, Apr 21, 2015 at 02:37:54AM +0200, Dario Faggioli wrote:
> On Fri, 2015-04-17 at 22:33 +0800, Chao Peng wrote:
> > Add dedicated one to show hardware information.
> >
> > [root@vmm-psr]xl psr-cmt-hwinfo
> > Cache Monitoring Technology (CMT):
> > Enabled : 1
> > Total RMID : 63
> > Supported monitor types:
> > cache-occupancy
> > total-mem-bandwidth
> > local-mem-bandwidth
> >
> Nice.
>
> > Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
>
> > --- a/tools/libxl/xl_cmdimpl.c
> > +++ b/tools/libxl/xl_cmdimpl.c
> > @@ -8014,6 +8014,36 @@ out:
> > }
> >
> > #ifdef LIBXL_HAVE_PSR_CMT
> > +static int psr_cmt_hwinfo(void)
> > +{
> > + int rc;
> > + int enabled;
> > + uint32_t total_rmid;
> > +
> >
> I think you should still have something like:
>
> SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cmt-hwinfo", 0) {
> /* No options */
> }
>
> Or `xl psr-cmt-hwinfo -h' wouldn't work, would it?
>
> With that done:
>
> Reviewed-by: Dario Faggioli <dario.faggioli@citrix.com>
>
Agreed, I will add it.
Chao
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH v5 11/13] tools/libxl: add command to show CMT hardware info
2015-04-21 0:37 ` Dario Faggioli
2015-04-21 9:42 ` Chao Peng
@ 2015-04-21 13:57 ` Ian Campbell
1 sibling, 0 replies; 33+ messages in thread
From: Ian Campbell @ 2015-04-21 13:57 UTC (permalink / raw)
To: Dario Faggioli
Cc: wei.liu2, keir, stefano.stabellini, andrew.cooper3, Ian.Jackson,
xen-devel, will.auld, JBeulich, Chao Peng, dgdegra
On Tue, 2015-04-21 at 02:37 +0200, Dario Faggioli wrote:
> On Fri, 2015-04-17 at 22:33 +0800, Chao Peng wrote:
> > Add dedicated one to show hardware information.
> >
> > [root@vmm-psr]xl psr-cmt-hwinfo
> > Cache Monitoring Technology (CMT):
> > Enabled : 1
> > Total RMID : 63
> > Supported monitor types:
> > cache-occupancy
> > total-mem-bandwidth
> > local-mem-bandwidth
> >
> Nice.
>
> > Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
>
> > --- a/tools/libxl/xl_cmdimpl.c
> > +++ b/tools/libxl/xl_cmdimpl.c
> > @@ -8014,6 +8014,36 @@ out:
> > }
> >
> > #ifdef LIBXL_HAVE_PSR_CMT
> > +static int psr_cmt_hwinfo(void)
> > +{
> > + int rc;
> > + int enabled;
> > + uint32_t total_rmid;
> > +
> >
> I think you should still have something like:
>
> SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cmt-hwinfo", 0) {
> /* No options */
> }
>
> Or `xl psr-cmt-hwinfo -h' wouldn't work, would it?
Yes, that is neded.
> With that done:
>
> Reviewed-by: Dario Faggioli <dario.faggioli@citrix.com>
Likewise:
Acked-by: Ian Campbell <ian.campbell@citrix.com>
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH v5 12/13] tools: add tools support for Intel CAT
2015-04-17 14:33 [PATCH v5 00/13] enable Cache Allocation Technology (CAT) for VMs Chao Peng
` (10 preceding siblings ...)
2015-04-17 14:33 ` [PATCH v5 11/13] tools/libxl: add command to show CMT hardware info Chao Peng
@ 2015-04-17 14:33 ` Chao Peng
2015-04-21 1:24 ` Dario Faggioli
2015-04-17 14:33 ` [PATCH v5 13/13] docs: add xl-psr.markdown Chao Peng
12 siblings, 1 reply; 33+ messages in thread
From: Chao Peng @ 2015-04-17 14:33 UTC (permalink / raw)
To: xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, will.auld, JBeulich, wei.liu2, dgdegra
This is the xc/xl changes to support Intel Cache Allocation
Technology(CAT). Two commands are introduced:
- xl psr-cat-hwinfo
Show CAT hardware information.
- xl psr-cat-cbm-set [-s socket] <domain> <cbm>
Set cache capacity bitmasks(CBM) for a domain.
- xl psr-cat-show <domain>
Show CAT domain information.
Examples:
[root@vmm-psr vmm]# xl psr-cat-hwinfo
Cache Allocation Technology (CAT):
Socket ID : 0
L3 Cache : 12288KB
Maximum COS : 15
CBM length : 12
Default CBM : 0xfff
[root@vmm-psr vmm]# xl psr-cat-cbm-set 0 0xff
[root@vmm-psr vmm]# xl psr-cat-show
Socket ID : 0
L3 Cache : 12288KB
Default CBM : 0xfff
ID NAME CBM
0 Domain-0 0xff
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
---
Changes in v5:
* Add psr-cat-hwinfo.
* Add libxl_psr_cat_info_list_free.
* malloc => libxl__malloc
* Other comments from Ian/Wei.
Changes in v4:
* Add example output in commit message.
* Make libxl__count_physical_sockets private to libxl_psr.c.
* Set errno in several error cases.
* Change libxl_psr_cat_get_l3_info to return all sockets information.
* Remove unused libxl_domain_info call.
Changes in v3:
* Add manpage.
* libxl_psr_cat_set/get_domain_data => libxl_psr_cat_set/get_cbm.
* Move libxl_count_physical_sockets into seperate patch.
* Support LIBXL_PSR_TARGET_ALL for libxl_psr_cat_set_cbm.
* Clean up the print codes.
---
docs/man/xl.pod.1 | 35 +++++++++
tools/libxc/include/xenctrl.h | 15 ++++
tools/libxc/xc_psr.c | 76 +++++++++++++++++++
tools/libxl/libxl.h | 26 +++++++
tools/libxl/libxl_psr.c | 170 ++++++++++++++++++++++++++++++++++++++++--
tools/libxl/libxl_types.idl | 10 +++
tools/libxl/xl.h | 5 ++
tools/libxl/xl_cmdimpl.c | 169 +++++++++++++++++++++++++++++++++++++++++
tools/libxl/xl_cmdtable.c | 19 +++++
9 files changed, 518 insertions(+), 7 deletions(-)
diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index 640788f..efc6599 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1517,6 +1517,41 @@ monitor types are:
=back
+=head1 CACHE ALLOCATION TECHNOLOGY
+
+Intel Broadwell and later server platforms offer capabilities to configure and
+make use of the Cache Allocation Technology (CAT) mechanisms, which enable more
+cache resources (i.e. L3 cache) to be made available for high priority
+applications. In Xen implementation, CAT is used to control cache allocation
+on VM basis. To enforce cache on a specific domain, just set capacity bitmasks
+(CBM) for the domain.
+
+=over 4
+
+=item B<psr-cat-hwinfo>
+
+Show CAT hardware information.
+
+=item B<psr-cat-cbm-set> [I<OPTIONS>] I<domain-id> I<cbm>
+
+Set cache capacity bitmasks(CBM) for a domain.
+
+B<OPTIONS>
+
+=over 4
+
+=item B<-s SOCKET>, B<--socket=SOCKET>
+
+Specify the socket to process, otherwise all sockets are processed.
+
+=back
+
+=item B<psr-cat-show> [I<domain-id>]
+
+Show CAT settings for a certain domain or all domains.
+
+=back
+
=head1 TO BE DOCUMENTED
We need better documentation for:
diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h
index 02d0db8..077cc1b 100644
--- a/tools/libxc/include/xenctrl.h
+++ b/tools/libxc/include/xenctrl.h
@@ -2696,6 +2696,12 @@ enum xc_psr_cmt_type {
XC_PSR_CMT_LOCAL_MEM_COUNT,
};
typedef enum xc_psr_cmt_type xc_psr_cmt_type;
+
+enum xc_psr_cat_type {
+ XC_PSR_CAT_L3_CBM = 1,
+};
+typedef enum xc_psr_cat_type xc_psr_cat_type;
+
int xc_psr_cmt_attach(xc_interface *xch, uint32_t domid);
int xc_psr_cmt_detach(xc_interface *xch, uint32_t domid);
int xc_psr_cmt_get_domain_rmid(xc_interface *xch, uint32_t domid,
@@ -2710,6 +2716,15 @@ int xc_psr_cmt_get_data(xc_interface *xch, uint32_t rmid, uint32_t cpu,
uint32_t psr_cmt_type, uint64_t *monitor_data,
uint64_t *tsc);
int xc_psr_cmt_enabled(xc_interface *xch);
+
+int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t domid,
+ xc_psr_cat_type type, uint32_t target,
+ uint64_t data);
+int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t domid,
+ xc_psr_cat_type type, uint32_t target,
+ uint64_t *data);
+int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
+ uint32_t *cos_max, uint32_t *cbm_len);
#endif
#endif /* XENCTRL_H */
diff --git a/tools/libxc/xc_psr.c b/tools/libxc/xc_psr.c
index e367a80..d8b3a51 100644
--- a/tools/libxc/xc_psr.c
+++ b/tools/libxc/xc_psr.c
@@ -248,6 +248,82 @@ int xc_psr_cmt_enabled(xc_interface *xch)
return 0;
}
+int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t domid,
+ xc_psr_cat_type type, uint32_t target,
+ uint64_t data)
+{
+ DECLARE_DOMCTL;
+ uint32_t cmd;
+
+ switch ( type )
+ {
+ case XC_PSR_CAT_L3_CBM:
+ cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM;
+ break;
+ default:
+ errno = EINVAL;
+ return -1;
+ }
+
+ domctl.cmd = XEN_DOMCTL_psr_cat_op;
+ domctl.domain = (domid_t)domid;
+ domctl.u.psr_cat_op.cmd = cmd;
+ domctl.u.psr_cat_op.target = target;
+ domctl.u.psr_cat_op.data = data;
+
+ return do_domctl(xch, &domctl);
+}
+
+int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t domid,
+ xc_psr_cat_type type, uint32_t target,
+ uint64_t *data)
+{
+ int rc;
+ DECLARE_DOMCTL;
+ uint32_t cmd;
+
+ switch ( type )
+ {
+ case XC_PSR_CAT_L3_CBM:
+ cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM;
+ break;
+ default:
+ errno = EINVAL;
+ return -1;
+ }
+
+ domctl.cmd = XEN_DOMCTL_psr_cat_op;
+ domctl.domain = (domid_t)domid;
+ domctl.u.psr_cat_op.cmd = cmd;
+ domctl.u.psr_cat_op.target = target;
+
+ rc = do_domctl(xch, &domctl);
+
+ if ( !rc )
+ *data = domctl.u.psr_cat_op.data;
+
+ return rc;
+}
+
+int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
+ uint32_t *cos_max, uint32_t *cbm_len)
+{
+ int rc;
+ DECLARE_SYSCTL;
+
+ sysctl.cmd = XEN_SYSCTL_psr_cat_op;
+ sysctl.u.psr_cat_op.cmd = XEN_SYSCTL_PSR_CAT_get_l3_info;
+ sysctl.u.psr_cat_op.target = socket;
+
+ rc = xc_sysctl(xch, &sysctl);
+ if ( !rc )
+ {
+ *cos_max = sysctl.u.psr_cat_op.u.l3_info.cos_max;
+ *cbm_len = sysctl.u.psr_cat_op.u.l3_info.cbm_len;
+ }
+
+ return rc;
+}
/*
* Local variables:
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 6bc75c5..e8e68f2 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -734,6 +734,13 @@ void libxl_mac_copy(libxl_ctx *ctx, libxl_mac *dst, libxl_mac *src);
* If this is defined, the Memory Bandwidth Monitoring feature is supported.
*/
#define LIBXL_HAVE_PSR_MBM 1
+
+/*
+ * LIBXL_HAVE_PSR_CAT
+ *
+ * If this is defined, the Cache Allocation Technology feature is supported.
+ */
+#define LIBXL_HAVE_PSR_CAT 1
#endif
typedef char **libxl_string_list;
@@ -1533,6 +1540,25 @@ int libxl_psr_cmt_get_sample(libxl_ctx *ctx,
uint64_t *tsc_r);
#endif
+#ifdef LIBXL_HAVE_PSR_CAT
+
+#define LIBXL_PSR_TARGET_ALL (~0U)
+int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid,
+ libxl_psr_cbm_type type, uint32_t target,
+ uint64_t cbm);
+int libxl_psr_cat_get_cbm(libxl_ctx *ctx, uint32_t domid,
+ libxl_psr_cbm_type type, uint32_t target,
+ uint64_t *cbm_r);
+
+/*
+ * On success, the function returns an array of elements in 'info',
+ * and the length in 'nr'.
+ */
+int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, libxl_psr_cat_info **info,
+ uint32_t *nr);
+void libxl_psr_cat_info_list_free(libxl_psr_cat_info *list, uint32_t nr);
+#endif
+
/* misc */
/* Each of these sets or clears the flag according to whether the
diff --git a/tools/libxl/libxl_psr.c b/tools/libxl/libxl_psr.c
index 3e1c792..9be9086 100644
--- a/tools/libxl/libxl_psr.c
+++ b/tools/libxl/libxl_psr.c
@@ -19,14 +19,37 @@
#define IA32_QM_CTR_ERROR_MASK (0x3ul << 62)
-static void libxl__psr_cmt_log_err_msg(libxl__gc *gc, int err)
+static void libxl__psr_log_err_msg(libxl__gc *gc, int err)
{
char *msg;
switch (err) {
case ENOSYS:
+ case EOPNOTSUPP:
msg = "unsupported operation";
break;
+ case ESRCH:
+ msg = "invalid domain ID";
+ break;
+ case EBADSLT:
+ msg = "socket is not supported";
+ break;
+ case EFAULT:
+ msg = "failed to exchange data with Xen";
+ break;
+ default:
+ msg = "unknown error";
+ break;
+ }
+
+ LOGE(ERROR, "%s", msg);
+}
+
+static void libxl__psr_cmt_log_err_msg(libxl__gc *gc, int err)
+{
+ char *msg;
+
+ switch (err) {
case ENODEV:
msg = "CMT is not supported in this system";
break;
@@ -39,15 +62,35 @@ static void libxl__psr_cmt_log_err_msg(libxl__gc *gc, int err)
case EUSERS:
msg = "no free RMID available";
break;
- case ESRCH:
- msg = "invalid domain ID";
+ default:
+ libxl__psr_log_err_msg(gc, err);
+ return;
+ }
+
+ LOGE(ERROR, "%s", msg);
+}
+
+static void libxl__psr_cat_log_err_msg(libxl__gc *gc, int err)
+{
+ char *msg;
+
+ switch (err) {
+ case ENODEV:
+ msg = "CAT is not supported in this system";
break;
- case EFAULT:
- msg = "failed to exchange data with Xen";
+ case ENOENT:
+ msg = "CAT is not enabled on the socket";
break;
- default:
- msg = "unknown error";
+ case EUSERS:
+ msg = "no free COS available";
+ break;
+ case EEXIST:
+ msg = "The same CBM is already set to this domain";
break;
+
+ default:
+ libxl__psr_log_err_msg(gc, err);
+ return;
}
LOGE(ERROR, "%s", msg);
@@ -73,6 +116,24 @@ static int libxl__pick_socket_cpu(libxl__gc *gc, uint32_t socketid)
return cpu;
}
+static int libxl__count_physical_sockets(libxl__gc *gc, uint32_t *sockets)
+{
+ int rc;
+ libxl_physinfo info;
+
+ libxl_physinfo_init(&info);
+
+ rc = libxl_get_physinfo(CTX, &info);
+ if (rc)
+ return rc;
+
+ *sockets = info.nr_cpus / info.threads_per_core
+ / info.cores_per_socket;
+
+ libxl_physinfo_dispose(&info);
+ return 0;
+}
+
int libxl_psr_cmt_attach(libxl_ctx *ctx, uint32_t domid)
{
GC_INIT(ctx);
@@ -247,6 +308,101 @@ out:
return rc;
}
+int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid,
+ libxl_psr_cbm_type type, uint32_t target,
+ uint64_t cbm)
+{
+ GC_INIT(ctx);
+ int r, rc = 0;
+ uint32_t i, nr_sockets;
+
+ if (target != LIBXL_PSR_TARGET_ALL) {
+ r = xc_psr_cat_set_domain_data(ctx->xch, domid, type, target, cbm);
+ if (r < 0) {
+ libxl__psr_cat_log_err_msg(gc, errno);
+ rc = ERROR_FAIL;
+ }
+ } else {
+ rc = libxl__count_physical_sockets(gc, &nr_sockets);
+ if (rc) {
+ LOGE(ERROR, "failed to get system socket count");
+ goto out;
+ }
+ for (i = 0; i < nr_sockets; i++) {
+ r = xc_psr_cat_set_domain_data(ctx->xch, domid, type, i, cbm);
+ if (r < 0) {
+ libxl__psr_cat_log_err_msg(gc, errno);
+ rc = ERROR_FAIL;
+ goto out;
+ }
+ }
+ }
+
+out:
+ GC_FREE;
+ return rc;
+}
+
+int libxl_psr_cat_get_cbm(libxl_ctx *ctx, uint32_t domid,
+ libxl_psr_cbm_type type, uint32_t target,
+ uint64_t *cbm_r)
+{
+ GC_INIT(ctx);
+ int r, rc = 0;
+
+ r = xc_psr_cat_get_domain_data(ctx->xch, domid, type, target, cbm_r);
+ if (r < 0) {
+ libxl__psr_cat_log_err_msg(gc, errno);
+ rc = ERROR_FAIL;
+ }
+
+ GC_FREE;
+ return rc;
+}
+
+int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, libxl_psr_cat_info **info,
+ uint32_t *nr)
+{
+ GC_INIT(ctx);
+ int rc, r;
+ uint32_t i, nr_sockets;
+ libxl_psr_cat_info *ptr;
+
+ rc = libxl__count_physical_sockets(gc, &nr_sockets);
+ if (rc) {
+ LOGE(ERROR, "failed to get system socket count");
+ goto out;
+ }
+
+ ptr = libxl__malloc(NOGC, nr_sockets * sizeof(libxl_psr_cat_info));
+
+ for (i = 0; i < nr_sockets; i++) {
+ r = xc_psr_cat_get_l3_info(ctx->xch, i, &ptr[i].cos_max,
+ &ptr[i].cbm_len);
+ if (r < 0) {
+ libxl__psr_cat_log_err_msg(gc, errno);
+ rc = ERROR_FAIL;
+ free(ptr);
+ goto out;
+ }
+ }
+
+ *info = ptr;
+ *nr = nr_sockets;
+out:
+ GC_FREE;
+ return rc;
+}
+
+void libxl_psr_cat_info_list_free(libxl_psr_cat_info *list, uint32_t nr)
+{
+ int i;
+
+ for (i = 0; i < nr; i++)
+ libxl_psr_cat_info_dispose(&list[i]);
+ free(list);
+}
+
/*
* Local variables:
* mode: C
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 0866433..ee8d469 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -709,3 +709,13 @@ libxl_psr_cmt_type = Enumeration("psr_cmt_type", [
(2, "TOTAL_MEM_COUNT"),
(3, "LOCAL_MEM_COUNT"),
])
+
+libxl_psr_cbm_type = Enumeration("psr_cbm_type", [
+ (0, "UNKNOWN"),
+ (1, "L3_CBM"),
+ ])
+
+libxl_psr_cat_info = Struct("psr_cat_info", [
+ ("cos_max", uint32),
+ ("cbm_len", uint32),
+ ])
diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h
index 1c526dc..c87b843 100644
--- a/tools/libxl/xl.h
+++ b/tools/libxl/xl.h
@@ -118,6 +118,11 @@ int main_psr_cmt_attach(int argc, char **argv);
int main_psr_cmt_detach(int argc, char **argv);
int main_psr_cmt_show(int argc, char **argv);
#endif
+#ifdef LIBXL_HAVE_PSR_CAT
+int main_psr_cat_hwinfo(int argc, char **argv);
+int main_psr_cat_cbm_set(int argc, char **argv);
+int main_psr_cat_show(int argc, char **argv);
+#endif
void help(const char *command);
diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c
index 4ab5e8b..29a1c22 100644
--- a/tools/libxl/xl_cmdimpl.c
+++ b/tools/libxl/xl_cmdimpl.c
@@ -8281,6 +8281,175 @@ int main_psr_cmt_show(int argc, char **argv)
}
#endif
+#ifdef LIBXL_HAVE_PSR_CAT
+static int psr_cat_hwinfo(void)
+{
+ int rc;
+ uint32_t socket, nr_sockets, l3_cache_size;
+ libxl_psr_cat_info *info;
+
+ printf("Cache Allocation Technology (CAT):\n");
+
+ rc = libxl_psr_cat_get_l3_info(ctx, &info, &nr_sockets);
+ if (rc) {
+ fprintf(stderr, "Failed to get cat info\n");
+ return rc;
+ }
+
+ for (socket = 0; socket < nr_sockets; socket++) {
+ rc = libxl_psr_cmt_get_l3_cache_size(ctx, socket, &l3_cache_size);
+ if (rc) {
+ fprintf(stderr, "Failed to get l3 cache size for socket:%d\n",
+ socket);
+ goto out;
+ }
+ printf("%-16s: %u\n", "Socket ID", socket);
+ printf("%-16s: %uKB\n", "L3 Cache", l3_cache_size);
+ printf("%-16s: %u\n", "Maximum COS", info->cos_max);
+ printf("%-16s: %u\n", "CBM length", info->cbm_len);
+ printf("%-16s: %#"PRIx64"\n", "Default CBM",
+ (1ul << info->cbm_len) - 1);
+ }
+
+out:
+ libxl_psr_cat_info_list_free(info, nr_sockets);
+ return rc;
+}
+
+static void psr_cat_print_one_domain_cbm(uint32_t domid, uint32_t socket)
+{
+ char *domain_name;
+ uint64_t cbm;
+
+ domain_name = libxl_domid_to_name(ctx, domid);
+ printf("%5d%25s", domid, domain_name);
+ free(domain_name);
+
+ if (!libxl_psr_cat_get_cbm(ctx, domid, LIBXL_PSR_CBM_TYPE_L3_CBM,
+ socket, &cbm))
+ printf("%#16"PRIx64, cbm);
+
+ printf("\n");
+}
+
+static int psr_cat_print_domain_cbm(uint32_t domid, uint32_t socket)
+{
+ int i, nr_domains;
+ libxl_dominfo *list;
+
+ if (domid != INVALID_DOMID) {
+ psr_cat_print_one_domain_cbm(domid, socket);
+ return 0;
+ }
+
+ if (!(list = libxl_list_domain(ctx, &nr_domains))) {
+ fprintf(stderr, "Failed to get domain list for cbm display\n");
+ return -1;
+ }
+
+ for (i = 0; i < nr_domains; i++)
+ psr_cat_print_one_domain_cbm(list[i].domid, socket);
+ libxl_dominfo_list_free(list, nr_domains);
+
+ return 0;
+}
+
+static int psr_cat_print_socket(uint32_t domid, uint32_t socket,
+ libxl_psr_cat_info *info)
+{
+ int rc;
+ uint32_t l3_cache_size;
+
+ rc = libxl_psr_cmt_get_l3_cache_size(ctx, socket, &l3_cache_size);
+ if (rc) {
+ fprintf(stderr, "Failed to get l3 cache size for socket:%d\n", socket);
+ return -1;
+ }
+
+ printf("%-16s: %u\n", "Socket ID", socket);
+ printf("%-16s: %uKB\n", "L3 Cache", l3_cache_size);
+ printf("%-16s: %#"PRIx64"\n", "Default CBM", (1ul << info->cbm_len) - 1);
+ printf("%5s%25s%16s\n", "ID", "NAME", "CBM");
+
+ return psr_cat_print_domain_cbm(domid, socket);
+}
+
+static int psr_cat_show(uint32_t domid)
+{
+ uint32_t socket, nr_sockets;
+ int rc;
+ libxl_psr_cat_info *info;
+
+ rc = libxl_psr_cat_get_l3_info(ctx, &info, &nr_sockets);
+ if (rc) {
+ fprintf(stderr, "Failed to get cat info\n");
+ return rc;
+ }
+
+ for (socket = 0; socket < nr_sockets; socket++) {
+ rc = psr_cat_print_socket(domid, socket, info + socket);
+ if (rc)
+ goto out;
+ }
+
+out:
+ libxl_psr_cat_info_list_free(info, nr_sockets);
+ return rc;
+}
+
+int main_psr_cat_hwinfo(int argc, char **argv)
+{
+ return psr_cat_hwinfo();
+}
+
+int main_psr_cat_cbm_set(int argc, char **argv)
+{
+ uint32_t domid;
+ uint32_t target = LIBXL_PSR_TARGET_ALL;
+ libxl_psr_cbm_type type = LIBXL_PSR_CBM_TYPE_L3_CBM;
+ uint64_t cbm;
+ int opt = 0;
+
+ static struct option opts[] = {
+ {"socket", 0, 0, 's'},
+ {0, 0, 0, 0}
+ };
+
+ SWITCH_FOREACH_OPT(opt, "s", opts, "psr-cat-cbm-set", 1) {
+ case 's':
+ target = strtol(optarg, NULL, 10);
+ break;
+ }
+
+ domid = find_domain(argv[optind]);
+ cbm = strtoll(argv[optind + 1], NULL , 0);
+
+ return libxl_psr_cat_set_cbm(ctx, domid, type, target, cbm);
+}
+
+int main_psr_cat_show(int argc, char **argv)
+{
+ int opt;
+ uint32_t domid;
+
+ SWITCH_FOREACH_OPT(opt, "", NULL, "psr-cat-show", 0) {
+ /* No options */
+ }
+
+ if (optind >= argc)
+ domid = INVALID_DOMID;
+ else if (optind == argc - 1)
+ domid = find_domain(argv[optind]);
+ else {
+ help("psr-cat-show");
+ return 2;
+ }
+
+ return psr_cat_show(domid);
+}
+
+#endif
+
/*
* Local variables:
* mode: C
diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c
index dc25d1f..a3816e5 100644
--- a/tools/libxl/xl_cmdtable.c
+++ b/tools/libxl/xl_cmdtable.c
@@ -549,6 +549,25 @@ struct cmd_spec cmd_table[] = {
"\"local_mem_bandwidth\": Show local memory bandwidth(KB/s)\n",
},
#endif
+#ifdef LIBXL_HAVE_PSR_CAT
+ { "psr-cat-hwinfo",
+ &main_psr_cat_hwinfo, 0, 1,
+ "Show hardware information for Cache Allocation Technology",
+ "",
+ },
+ { "psr-cat-cbm-set",
+ &main_psr_cat_cbm_set, 0, 1,
+ "Set cache capacity bitmasks(CBM) for a domain",
+ "-s <socket> Specify the socket to process, otherwise all sockets are processed\n"
+ "<Domain> <CBM>",
+ },
+ { "psr-cat-show",
+ &main_psr_cat_show, 0, 1,
+ "Show Cache Allocation Technology information",
+ "<Domain>",
+ },
+
+#endif
};
int cmdtable_len = sizeof(cmd_table)/sizeof(struct cmd_spec);
--
1.9.1
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH v5 12/13] tools: add tools support for Intel CAT
2015-04-17 14:33 ` [PATCH v5 12/13] tools: add tools support for Intel CAT Chao Peng
@ 2015-04-21 1:24 ` Dario Faggioli
2015-04-21 9:49 ` Chao Peng
0 siblings, 1 reply; 33+ messages in thread
From: Dario Faggioli @ 2015-04-21 1:24 UTC (permalink / raw)
To: Chao Peng
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, xen-devel, will.auld, JBeulich, wei.liu2, dgdegra
[-- Attachment #1.1: Type: text/plain, Size: 5149 bytes --]
On Fri, 2015-04-17 at 22:33 +0800, Chao Peng wrote:
> This is the xc/xl changes to support Intel Cache Allocation
> Technology(CAT). Two commands are introduced:
> - xl psr-cat-hwinfo
> Show CAT hardware information.
> Examples:
> [root@vmm-psr vmm]# xl psr-cat-hwinfo
> Cache Allocation Technology (CAT):
> Socket ID : 0
> L3 Cache : 12288KB
> Maximum COS : 15
> CBM length : 12
> Default CBM : 0xfff
>
Or, you can rename the psr-cmt-hwinfo command, added in the previous
patch, to 'psr-hwinfo' and make it accept options, e.g.,
"-m, --cmt show Cache Monitoring Technology (CMT) hardware info"
"-c, --cat show Cache Allocation Technology (CAT) hardware info"
By default (i.e., no options provided), it can just print all the hw
info.
Not a big deal, but I think that would make a better command line
interface. Tools' maintainers' call, I guess.
> --- a/tools/libxc/include/xenctrl.h
> +++ b/tools/libxc/include/xenctrl.h
> +
> +int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t domid,
> + xc_psr_cat_type type, uint32_t target,
> + uint64_t data);
> +int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t domid,
> + xc_psr_cat_type type, uint32_t target,
> + uint64_t *data);
>
So, for this twos, 'target' is the socket you want to act on.
> +int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
> + uint32_t *cos_max, uint32_t *cbm_len);
>
While here you use 'socket', to mean the same thing.
That looks rather inconsistent. Since it's a socket we are talking
about, why not 'socket' everywhere?
> --- a/tools/libxl/libxl.h
> +++ b/tools/libxl/libxl.h
> +#ifdef LIBXL_HAVE_PSR_CAT
> +
> +#define LIBXL_PSR_TARGET_ALL (~0U)
> +int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid,
> + libxl_psr_cbm_type type, uint32_t target,
> + uint64_t cbm);
> +int libxl_psr_cat_get_cbm(libxl_ctx *ctx, uint32_t domid,
> + libxl_psr_cbm_type type, uint32_t target,
> + uint64_t *cbm_r);
> +
The same applies here: I'd rename taget to socket.
Then there's that LIBXL_PSR_TARGET_ALL. So, target (or socket, if you'll
rename it) is an integer representing _which_one_ socket to act on.
However, there is a special value to mean "all sockets".
Another possibility would be to offer an API that "natively" allows for
operating on multiple sockets, by using libxl_bitmap-s, as it happens in
many other places, in libxl itself (e.g., setting and getting vcpu
affinity).
That means target would become a libxl_bitmap, and, in the
implementation, you'll apply the operation on all the sockets
corresponding to a set bit in there. Only one bit set means just that
socket, all bits means all sockets.
This looks like a better interface to me (no need for special ~0U
values), it'd make the implementation more linear, and is more
consistent with how other similar situations are handled in libxl.
However, I appreciate that one may find it overkill... I guess it
depends whether we expect the prevalent usage pattern to be almost
always about single sockets --and maybe on all sockets, from time to
time-- or if we see value in being able to specify more than one and
less than all sockets.
For instance, now that we have vNUMA, if a domain has 4 vNUMA nodes,
each one mapped to a physical NUMA node, it looks to me like it would
make sense to set CAT to, say, 0x0F, on the sockets corresponding to the
physical NODEs. With the interface in this patch, that would require
calling libxl_psr_cat_set_cbm() 4 times, with the libxl_bitmap approach,
just once, after setting up the bitmap properly.
Thoughts?
> --- a/tools/libxl/libxl_psr.c
> +++ b/tools/libxl/libxl_psr.c
> +int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, libxl_psr_cat_info **info,
> + uint32_t *nr)
> +{
> + GC_INIT(ctx);
> + int rc, r;
> + uint32_t i, nr_sockets;
> + libxl_psr_cat_info *ptr;
> +
> + rc = libxl__count_physical_sockets(gc, &nr_sockets);
> + if (rc) {
> + LOGE(ERROR, "failed to get system socket count");
> + goto out;
> + }
> +
> + ptr = libxl__malloc(NOGC, nr_sockets * sizeof(libxl_psr_cat_info));
> +
> + for (i = 0; i < nr_sockets; i++) {
> + r = xc_psr_cat_get_l3_info(ctx->xch, i, &ptr[i].cos_max,
> + &ptr[i].cbm_len);
> + if (r < 0) {
>
why not just:
if (xc_psr_cat_get_l3_info(ctx->xch, i, &ptr[i].cos_max,
&ptr[i].cbm_len)) {
> + libxl__psr_cat_log_err_msg(gc, errno);
> + rc = ERROR_FAIL;
> + free(ptr);
> + goto out;
> + }
> + }
> +
I mean, you don't need to save the actual return value from libxc, do
you? The same applies to other places down in the patch.
Regards,
Dario
[-- Attachment #1.2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 181 bytes --]
[-- Attachment #2: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH v5 12/13] tools: add tools support for Intel CAT
2015-04-21 1:24 ` Dario Faggioli
@ 2015-04-21 9:49 ` Chao Peng
2015-04-21 14:01 ` Ian Campbell
2015-04-21 15:15 ` Dario Faggioli
0 siblings, 2 replies; 33+ messages in thread
From: Chao Peng @ 2015-04-21 9:49 UTC (permalink / raw)
To: Dario Faggioli
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, xen-devel, will.auld, JBeulich, wei.liu2, dgdegra
On Tue, Apr 21, 2015 at 03:24:37AM +0200, Dario Faggioli wrote:
> On Fri, 2015-04-17 at 22:33 +0800, Chao Peng wrote:
> > This is the xc/xl changes to support Intel Cache Allocation
> > Technology(CAT). Two commands are introduced:
> > - xl psr-cat-hwinfo
> > Show CAT hardware information.
>
> > Examples:
> > [root@vmm-psr vmm]# xl psr-cat-hwinfo
> > Cache Allocation Technology (CAT):
> > Socket ID : 0
> > L3 Cache : 12288KB
> > Maximum COS : 15
> > CBM length : 12
> > Default CBM : 0xfff
> >
> Or, you can rename the psr-cmt-hwinfo command, added in the previous
> patch, to 'psr-hwinfo' and make it accept options, e.g.,
>
> "-m, --cmt show Cache Monitoring Technology (CMT) hardware info"
> "-c, --cat show Cache Allocation Technology (CAT) hardware info"
>
> By default (i.e., no options provided), it can just print all the hw
> info.
>
> Not a big deal, but I think that would make a better command line
> interface. Tools' maintainers' call, I guess.
Thanks for suggestion.
>
> > --- a/tools/libxc/include/xenctrl.h
> > +++ b/tools/libxc/include/xenctrl.h
>
> > +
> > +int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t domid,
> > + xc_psr_cat_type type, uint32_t target,
> > + uint64_t data);
> > +int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t domid,
> > + xc_psr_cat_type type, uint32_t target,
> > + uint64_t *data);
> >
> So, for this twos, 'target' is the socket you want to act on.
>
> > +int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
> > + uint32_t *cos_max, uint32_t *cbm_len);
> >
> While here you use 'socket', to mean the same thing.
>
> That looks rather inconsistent. Since it's a socket we are talking
> about, why not 'socket' everywhere?
The idea behind here is: All the places that appear as 'target' imply
there are possible values other than just socket (e.g. considering L2
Cache Allocation in the future). So 'target' is always paired with a
'psr_cat_type' parameter. For routines that only work for L3 (e.g.
xc_psr_cat_get_l3_info) then 'socket' is used. I admit that it looks
inconsistent, perhaps rename all 'socket' to 'target'?
>
> > --- a/tools/libxl/libxl.h
> > +++ b/tools/libxl/libxl.h
>
> > +#ifdef LIBXL_HAVE_PSR_CAT
> > +
> > +#define LIBXL_PSR_TARGET_ALL (~0U)
> > +int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid,
> > + libxl_psr_cbm_type type, uint32_t target,
> > + uint64_t cbm);
> > +int libxl_psr_cat_get_cbm(libxl_ctx *ctx, uint32_t domid,
> > + libxl_psr_cbm_type type, uint32_t target,
> > + uint64_t *cbm_r);
> > +
> The same applies here: I'd rename taget to socket.
>
> Then there's that LIBXL_PSR_TARGET_ALL. So, target (or socket, if you'll
> rename it) is an integer representing _which_one_ socket to act on.
> However, there is a special value to mean "all sockets".
>
> Another possibility would be to offer an API that "natively" allows for
> operating on multiple sockets, by using libxl_bitmap-s, as it happens in
> many other places, in libxl itself (e.g., setting and getting vcpu
> affinity).
>
> That means target would become a libxl_bitmap, and, in the
> implementation, you'll apply the operation on all the sockets
> corresponding to a set bit in there. Only one bit set means just that
> socket, all bits means all sockets.
>
> This looks like a better interface to me (no need for special ~0U
> values), it'd make the implementation more linear, and is more
> consistent with how other similar situations are handled in libxl.
> However, I appreciate that one may find it overkill... I guess it
> depends whether we expect the prevalent usage pattern to be almost
> always about single sockets --and maybe on all sockets, from time to
> time-- or if we see value in being able to specify more than one and
> less than all sockets.
>
> For instance, now that we have vNUMA, if a domain has 4 vNUMA nodes,
> each one mapped to a physical NUMA node, it looks to me like it would
> make sense to set CAT to, say, 0x0F, on the sockets corresponding to the
> physical NODEs. With the interface in this patch, that would require
> calling libxl_psr_cat_set_cbm() 4 times, with the libxl_bitmap approach,
> just once, after setting up the bitmap properly.
>
> Thoughts?
I do like this suggestion and I have ever considered it actually. The
only thing prevents me is that we need an extra _get_socket_count in xl
for TARGET_ALL case. So libxl__count_physical_sockets is needed to be
public. If Ian/Wei have no concerns for this, then I'm glad to do this.
>
> > --- a/tools/libxl/libxl_psr.c
> > +++ b/tools/libxl/libxl_psr.c
>
> > +int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, libxl_psr_cat_info **info,
> > + uint32_t *nr)
> > +{
> > + GC_INIT(ctx);
> > + int rc, r;
> > + uint32_t i, nr_sockets;
> > + libxl_psr_cat_info *ptr;
> > +
> > + rc = libxl__count_physical_sockets(gc, &nr_sockets);
> > + if (rc) {
> > + LOGE(ERROR, "failed to get system socket count");
> > + goto out;
> > + }
> > +
> > + ptr = libxl__malloc(NOGC, nr_sockets * sizeof(libxl_psr_cat_info));
> > +
> > + for (i = 0; i < nr_sockets; i++) {
> > + r = xc_psr_cat_get_l3_info(ctx->xch, i, &ptr[i].cos_max,
> > + &ptr[i].cbm_len);
> > + if (r < 0) {
> >
> why not just:
>
> if (xc_psr_cat_get_l3_info(ctx->xch, i, &ptr[i].cos_max,
> &ptr[i].cbm_len)) {
>
> > + libxl__psr_cat_log_err_msg(gc, errno);
> > + rc = ERROR_FAIL;
> > + free(ptr);
> > + goto out;
> > + }
> > + }
> > +
> I mean, you don't need to save the actual return value from libxc, do
> you? The same applies to other places down in the patch.
Agreed, thanks.
Chao
^ permalink raw reply [flat|nested] 33+ messages in thread* Re: [PATCH v5 12/13] tools: add tools support for Intel CAT
2015-04-21 9:49 ` Chao Peng
@ 2015-04-21 14:01 ` Ian Campbell
2015-04-21 14:39 ` Dario Faggioli
2015-04-21 15:15 ` Dario Faggioli
1 sibling, 1 reply; 33+ messages in thread
From: Ian Campbell @ 2015-04-21 14:01 UTC (permalink / raw)
To: Chao Peng
Cc: keir, stefano.stabellini, andrew.cooper3, Dario Faggioli,
Ian.Jackson, xen-devel, will.auld, JBeulich, wei.liu2, dgdegra
On Tue, 2015-04-21 at 17:49 +0800, Chao Peng wrote:
> On Tue, Apr 21, 2015 at 03:24:37AM +0200, Dario Faggioli wrote:
> > On Fri, 2015-04-17 at 22:33 +0800, Chao Peng wrote:
> > > This is the xc/xl changes to support Intel Cache Allocation
> > > Technology(CAT). Two commands are introduced:
> > > - xl psr-cat-hwinfo
> > > Show CAT hardware information.
> >
> > > Examples:
> > > [root@vmm-psr vmm]# xl psr-cat-hwinfo
> > > Cache Allocation Technology (CAT):
> > > Socket ID : 0
> > > L3 Cache : 12288KB
> > > Maximum COS : 15
> > > CBM length : 12
> > > Default CBM : 0xfff
> > >
> > Or, you can rename the psr-cmt-hwinfo command, added in the previous
> > patch, to 'psr-hwinfo' and make it accept options, e.g.,
> >
> > "-m, --cmt show Cache Monitoring Technology (CMT) hardware info"
> > "-c, --cat show Cache Allocation Technology (CAT) hardware info"
> >
> > By default (i.e., no options provided), it can just print all the hw
> > info.
> >
> > Not a big deal, but I think that would make a better command line
> > interface. Tools' maintainers' call, I guess.
>
> Thanks for suggestion.
FWIW I think this is a good idea.
> > > --- a/tools/libxl/libxl.h
> > > +++ b/tools/libxl/libxl.h
> >
> > > +#ifdef LIBXL_HAVE_PSR_CAT
> > > +
> > > +#define LIBXL_PSR_TARGET_ALL (~0U)
> > > +int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid,
> > > + libxl_psr_cbm_type type, uint32_t target,
> > > + uint64_t cbm);
> > > +int libxl_psr_cat_get_cbm(libxl_ctx *ctx, uint32_t domid,
> > > + libxl_psr_cbm_type type, uint32_t target,
> > > + uint64_t *cbm_r);
> > > +
> > The same applies here: I'd rename taget to socket.
> >
> > Then there's that LIBXL_PSR_TARGET_ALL. So, target (or socket, if you'll
> > rename it) is an integer representing _which_one_ socket to act on.
> > However, there is a special value to mean "all sockets".
> >
> > Another possibility would be to offer an API that "natively" allows for
> > operating on multiple sockets, by using libxl_bitmap-s, as it happens in
> > many other places, in libxl itself (e.g., setting and getting vcpu
> > affinity).
> >
> > That means target would become a libxl_bitmap, and, in the
> > implementation, you'll apply the operation on all the sockets
> > corresponding to a set bit in there. Only one bit set means just that
> > socket, all bits means all sockets.
> >
> > This looks like a better interface to me (no need for special ~0U
> > values), it'd make the implementation more linear, and is more
> > consistent with how other similar situations are handled in libxl.
> > However, I appreciate that one may find it overkill... I guess it
> > depends whether we expect the prevalent usage pattern to be almost
> > always about single sockets --and maybe on all sockets, from time to
> > time-- or if we see value in being able to specify more than one and
> > less than all sockets.
> >
> > For instance, now that we have vNUMA, if a domain has 4 vNUMA nodes,
> > each one mapped to a physical NUMA node, it looks to me like it would
> > make sense to set CAT to, say, 0x0F, on the sockets corresponding to the
> > physical NODEs. With the interface in this patch, that would require
> > calling libxl_psr_cat_set_cbm() 4 times, with the libxl_bitmap approach,
> > just once, after setting up the bitmap properly.
> >
> > Thoughts?
>
> I do like this suggestion and I have ever considered it actually. The
> only thing prevents me is that we need an extra _get_socket_count in xl
> for TARGET_ALL case. So libxl__count_physical_sockets is needed to be
> public. If Ian/Wei have no concerns for this, then I'm glad to do this.
I don't think you need libxl__count_physical_sockets for this, the
existing xl code for similar things just uses libxl_bitmap_set_any to
handle the all case.
With that in mind Dario's suggestion does seem like a good improvement
to the interface.
Ian.
^ permalink raw reply [flat|nested] 33+ messages in thread
* Re: [PATCH v5 12/13] tools: add tools support for Intel CAT
2015-04-21 14:01 ` Ian Campbell
@ 2015-04-21 14:39 ` Dario Faggioli
2015-04-22 13:09 ` Chao Peng
0 siblings, 1 reply; 33+ messages in thread
From: Dario Faggioli @ 2015-04-21 14:39 UTC (permalink / raw)
To: Ian Campbell
Cc: wei.liu2, keir, stefano.stabellini, andrew.cooper3, Ian.Jackson,
xen-devel, will.auld, JBeulich, Chao Peng, dgdegra
[-- Attachment #1.1: Type: text/plain, Size: 1203 bytes --]
On Tue, 2015-04-21 at 15:01 +0100, Ian Campbell wrote:
> On Tue, 2015-04-21 at 17:49 +0800, Chao Peng wrote:
> > I do like this suggestion and I have ever considered it actually. The
> > only thing prevents me is that we need an extra _get_socket_count in xl
> > for TARGET_ALL case. So libxl__count_physical_sockets is needed to be
> > public. If Ian/Wei have no concerns for this, then I'm glad to do this.
>
> I don't think you need libxl__count_physical_sockets for this,
>
I also don't think that is necessary.
> the
> existing xl code for similar things just uses libxl_bitmap_set_any to
> handle the all case.
>
That or, if you want to do something similar to what we have in *libxl*
now you can look at the implementation of libxl_cpu_bitmap_alloc() (or
libxl_node_bitmap_alloc() ) and define yours
libxl_socket_bitmap_alloc().
In there, libxl_get_max_cpus() ( *_nodes()) is used, internally, to
determine the size of the bitmap, if the user did not provide it
explicitly. In your case, you can use libxl__count_physical_sockets() to
the same effect, without the need for it to be public.
But then again, that is probably even not necessary.
Regards,
Dario
[-- Attachment #1.2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 181 bytes --]
[-- Attachment #2: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 33+ messages in thread
* Re: [PATCH v5 12/13] tools: add tools support for Intel CAT
2015-04-21 14:39 ` Dario Faggioli
@ 2015-04-22 13:09 ` Chao Peng
0 siblings, 0 replies; 33+ messages in thread
From: Chao Peng @ 2015-04-22 13:09 UTC (permalink / raw)
To: Dario Faggioli
Cc: keir, Ian Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, xen-devel, will.auld, JBeulich, wei.liu2, dgdegra
On Tue, Apr 21, 2015 at 04:39:05PM +0200, Dario Faggioli wrote:
> On Tue, 2015-04-21 at 15:01 +0100, Ian Campbell wrote:
> > On Tue, 2015-04-21 at 17:49 +0800, Chao Peng wrote:
>
> > > I do like this suggestion and I have ever considered it actually. The
> > > only thing prevents me is that we need an extra _get_socket_count in xl
> > > for TARGET_ALL case. So libxl__count_physical_sockets is needed to be
> > > public. If Ian/Wei have no concerns for this, then I'm glad to do this.
> >
> > I don't think you need libxl__count_physical_sockets for this,
> >
> I also don't think that is necessary.
>
> > the
> > existing xl code for similar things just uses libxl_bitmap_set_any to
> > handle the all case.
> >
> That or, if you want to do something similar to what we have in *libxl*
> now you can look at the implementation of libxl_cpu_bitmap_alloc() (or
> libxl_node_bitmap_alloc() ) and define yours
> libxl_socket_bitmap_alloc().
>
> In there, libxl_get_max_cpus() ( *_nodes()) is used, internally, to
> determine the size of the bitmap, if the user did not provide it
> explicitly. In your case, you can use libxl__count_physical_sockets() to
> the same effect, without the need for it to be public.
>
> But then again, that is probably even not necessary.
>
I tend to add libxl_socket_bitmap_alloc(), otherwise a prediefined
MAX_SOCKETS is needed.
Chao
^ permalink raw reply [flat|nested] 33+ messages in thread
* Re: [PATCH v5 12/13] tools: add tools support for Intel CAT
2015-04-21 9:49 ` Chao Peng
2015-04-21 14:01 ` Ian Campbell
@ 2015-04-21 15:15 ` Dario Faggioli
2015-04-22 13:37 ` Chao Peng
1 sibling, 1 reply; 33+ messages in thread
From: Dario Faggioli @ 2015-04-21 15:15 UTC (permalink / raw)
To: Chao Peng
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, xen-devel, will.auld, JBeulich, wei.liu2, dgdegra
[-- Attachment #1.1: Type: text/plain, Size: 3985 bytes --]
On Tue, 2015-04-21 at 17:49 +0800, Chao Peng wrote:
> On Tue, Apr 21, 2015 at 03:24:37AM +0200, Dario Faggioli wrote:
> > On Fri, 2015-04-17 at 22:33 +0800, Chao Peng wrote:
> > > This is the xc/xl changes to support Intel Cache Allocation
> > > Technology(CAT). Two commands are introduced:
> > > - xl psr-cat-hwinfo
> > > Show CAT hardware information.
> >
> > > Examples:
> > > [root@vmm-psr vmm]# xl psr-cat-hwinfo
> > > Cache Allocation Technology (CAT):
> > > Socket ID : 0
> > > L3 Cache : 12288KB
> > > Maximum COS : 15
> > > CBM length : 12
> > > Default CBM : 0xfff
> > >
> > Or, you can rename the psr-cmt-hwinfo command, added in the previous
> > patch, to 'psr-hwinfo' and make it accept options, e.g.,
> >
> > "-m, --cmt show Cache Monitoring Technology (CMT) hardware info"
> > "-c, --cat show Cache Allocation Technology (CAT) hardware info"
> >
> > By default (i.e., no options provided), it can just print all the hw
> > info.
> >
> > Not a big deal, but I think that would make a better command line
> > interface. Tools' maintainers' call, I guess.
>
> Thanks for suggestion.
> >
> > > --- a/tools/libxc/include/xenctrl.h
> > > +++ b/tools/libxc/include/xenctrl.h
> >
> > > +
> > > +int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t domid,
> > > + xc_psr_cat_type type, uint32_t target,
> > > + uint64_t data);
> > > +int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t domid,
> > > + xc_psr_cat_type type, uint32_t target,
> > > + uint64_t *data);
> > >
> > So, for this twos, 'target' is the socket you want to act on.
> >
> > > +int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
> > > + uint32_t *cos_max, uint32_t *cbm_len);
> > >
> > While here you use 'socket', to mean the same thing.
> >
> > That looks rather inconsistent. Since it's a socket we are talking
> > about, why not 'socket' everywhere?
>
>
> The idea behind here is: All the places that appear as 'target' imply
> there are possible values other than just socket (e.g. considering L2
> Cache Allocation in the future). So 'target' is always paired with a
> 'psr_cat_type' parameter.
>
Mmm... I understand your concerns. So, sticking to the future L2 CAT
support example, what would 'target' mean in that case, a pCPU? I'll
have to be something that makes it possible to 'identify' an L2, as much
as socket identifies an L3... Is this the case?
I'm not sure. My own concerns are that, if I look at the prototypes of
this functions, it's not that evident what values should I use for the
type and target parameters, whether there are constraints/relationships
among them, etc. That applies to both the xc_* functions above and the
libxl_* functions below, IMO.
Libxc is not a stable interface, so we could even just forget about
this, design this very interface _only_ for what we have now and deal
with different CBM types when we'll be introducing them. However, libxl
*does* have a stable API, so we still need to solve the issue at that
level.
> For routines that only work for L3 (e.g.
> xc_psr_cat_get_l3_info) then 'socket' is used. I admit that it looks
> inconsistent, perhaps rename all 'socket' to 'target'?
>
No, IMO, that is one "good inconsistency", as it allows, at least for
that function, to easily figure out what one should pass to the function
by means of that parameter! :-)
I really am not sure, and probably would have to know in what way(s)
'target' would change its meaning, depending on the value of 'type' (as
asked above)... Probably what I'd do is leave parameters names as they
are, but write a few doc-comments to explain how to use them, especially
at the libxl level (libxc is a lot less critical, from this respect, I
think).
Regards,
Dario
[-- Attachment #1.2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 181 bytes --]
[-- Attachment #2: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
^ permalink raw reply [flat|nested] 33+ messages in thread
* Re: [PATCH v5 12/13] tools: add tools support for Intel CAT
2015-04-21 15:15 ` Dario Faggioli
@ 2015-04-22 13:37 ` Chao Peng
0 siblings, 0 replies; 33+ messages in thread
From: Chao Peng @ 2015-04-22 13:37 UTC (permalink / raw)
To: Dario Faggioli
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, xen-devel, will.auld, JBeulich, wei.liu2, dgdegra
On Tue, Apr 21, 2015 at 05:15:15PM +0200, Dario Faggioli wrote:
> On Tue, 2015-04-21 at 17:49 +0800, Chao Peng wrote:
> > On Tue, Apr 21, 2015 at 03:24:37AM +0200, Dario Faggioli wrote:
> > > On Fri, 2015-04-17 at 22:33 +0800, Chao Peng wrote:
> > > > This is the xc/xl changes to support Intel Cache Allocation
> > > > Technology(CAT). Two commands are introduced:
> > > > - xl psr-cat-hwinfo
> > > > Show CAT hardware information.
> > >
> > > > Examples:
> > > > [root@vmm-psr vmm]# xl psr-cat-hwinfo
> > > > Cache Allocation Technology (CAT):
> > > > Socket ID : 0
> > > > L3 Cache : 12288KB
> > > > Maximum COS : 15
> > > > CBM length : 12
> > > > Default CBM : 0xfff
> > > >
> > > Or, you can rename the psr-cmt-hwinfo command, added in the previous
> > > patch, to 'psr-hwinfo' and make it accept options, e.g.,
> > >
> > > "-m, --cmt show Cache Monitoring Technology (CMT) hardware info"
> > > "-c, --cat show Cache Allocation Technology (CAT) hardware info"
> > >
> > > By default (i.e., no options provided), it can just print all the hw
> > > info.
> > >
> > > Not a big deal, but I think that would make a better command line
> > > interface. Tools' maintainers' call, I guess.
> >
> > Thanks for suggestion.
> > >
> > > > --- a/tools/libxc/include/xenctrl.h
> > > > +++ b/tools/libxc/include/xenctrl.h
> > >
> > > > +
> > > > +int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t domid,
> > > > + xc_psr_cat_type type, uint32_t target,
> > > > + uint64_t data);
> > > > +int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t domid,
> > > > + xc_psr_cat_type type, uint32_t target,
> > > > + uint64_t *data);
> > > >
> > > So, for this twos, 'target' is the socket you want to act on.
> > >
> > > > +int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
> > > > + uint32_t *cos_max, uint32_t *cbm_len);
> > > >
> > > While here you use 'socket', to mean the same thing.
> > >
> > > That looks rather inconsistent. Since it's a socket we are talking
> > > about, why not 'socket' everywhere?
> >
> >
> > The idea behind here is: All the places that appear as 'target' imply
> > there are possible values other than just socket (e.g. considering L2
> > Cache Allocation in the future). So 'target' is always paired with a
> > 'psr_cat_type' parameter.
> >
> Mmm... I understand your concerns. So, sticking to the future L2 CAT
> support example, what would 'target' mean in that case, a pCPU? I'll
> have to be something that makes it possible to 'identify' an L2, as much
> as socket identifies an L3... Is this the case?
For L2, it's likely to be the cpu or core id, but not socket.
>
> I'm not sure. My own concerns are that, if I look at the prototypes of
> this functions, it's not that evident what values should I use for the
> type and target parameters, whether there are constraints/relationships
> among them, etc. That applies to both the xc_* functions above and the
> libxl_* functions below, IMO.
>
> Libxc is not a stable interface, so we could even just forget about
> this, design this very interface _only_ for what we have now and deal
> with different CBM types when we'll be introducing them. However, libxl
> *does* have a stable API, so we still need to solve the issue at that
> level.
>
> > For routines that only work for L3 (e.g.
> > xc_psr_cat_get_l3_info) then 'socket' is used. I admit that it looks
> > inconsistent, perhaps rename all 'socket' to 'target'?
> >
> No, IMO, that is one "good inconsistency", as it allows, at least for
> that function, to easily figure out what one should pass to the function
> by means of that parameter! :-)
>
> I really am not sure, and probably would have to know in what way(s)
> 'target' would change its meaning, depending on the value of 'type' (as
> asked above)... Probably what I'd do is leave parameters names as they
> are, but write a few doc-comments to explain how to use them, especially
> at the libxl level (libxc is a lot less critical, from this respect, I
> think).
OK, I can add some comments.
Thanks,
Chao
^ permalink raw reply [flat|nested] 33+ messages in thread
* [PATCH v5 13/13] docs: add xl-psr.markdown
2015-04-17 14:33 [PATCH v5 00/13] enable Cache Allocation Technology (CAT) for VMs Chao Peng
` (11 preceding siblings ...)
2015-04-17 14:33 ` [PATCH v5 12/13] tools: add tools support for Intel CAT Chao Peng
@ 2015-04-17 14:33 ` Chao Peng
2015-04-21 14:05 ` Ian Campbell
12 siblings, 1 reply; 33+ messages in thread
From: Chao Peng @ 2015-04-17 14:33 UTC (permalink / raw)
To: xen-devel
Cc: keir, Ian.Campbell, stefano.stabellini, andrew.cooper3,
Ian.Jackson, will.auld, JBeulich, wei.liu2, dgdegra
Add document to introduce basic concepts and terms in PSR family
technologies and the xl interfaces.
Signed-off-by: Chao Peng <chao.p.peng@linux.intel.com>
---
Changes in v5:
* Address comments from Andrew/Ian.
---
docs/man/xl.pod.1 | 10 +++-
docs/misc/xl-psr.markdown | 134 ++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 143 insertions(+), 1 deletion(-)
create mode 100644 docs/misc/xl-psr.markdown
diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index efc6599..b024d45 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1493,6 +1493,9 @@ occupancy monitoring share the same set of underlying monitoring service. Once
a domain is attached to the monitoring service, monitoring data can be showed
for any of these monitoring types.
+See L<http://xenbits.xen.org/docs/unstable/misc/xl-psr.html> for more
+information.
+
=over 4
=item B<psr-cmt-hwinfo>
@@ -1526,6 +1529,9 @@ applications. In Xen implementation, CAT is used to control cache allocation
on VM basis. To enforce cache on a specific domain, just set capacity bitmasks
(CBM) for the domain.
+See L<http://xenbits.xen.org/docs/unstable/misc/xl-psr.html> for more
+information.
+
=over 4
=item B<psr-cat-hwinfo>
@@ -1534,7 +1540,8 @@ Show CAT hardware information.
=item B<psr-cat-cbm-set> [I<OPTIONS>] I<domain-id> I<cbm>
-Set cache capacity bitmasks(CBM) for a domain.
+Set cache capacity bitmasks(CBM) for a domain. For how to specify I<cbm>
+please refer to the link above.
B<OPTIONS>
@@ -1575,6 +1582,7 @@ And the following documents on the xen.org website:
L<http://xenbits.xen.org/docs/unstable/misc/xl-network-configuration.html>
L<http://xenbits.xen.org/docs/unstable/misc/xl-disk-configuration.txt>
L<http://xenbits.xen.org/docs/unstable/misc/xsm-flask.txt>
+L<http://xenbits.xen.org/docs/unstable/misc/xl-psr.html>
For systems that don't automatically bring CPU online:
diff --git a/docs/misc/xl-psr.markdown b/docs/misc/xl-psr.markdown
new file mode 100644
index 0000000..d167b84
--- /dev/null
+++ b/docs/misc/xl-psr.markdown
@@ -0,0 +1,134 @@
+# Intel Platform Shared Resource Monitoring/Control in xl
+
+This document introduces Intel Platform Shared Resource Monitoring/Control
+technologies, their basic concepts and the xl interfaces.
+
+## Cache Monitoring Technology (CMT)
+
+Cache Monitoring Technology (CMT) is a new feature available on Intel Haswell
+and later server platforms that allows an OS or Hypervisor/VMM to determine
+the usage of cache(currently only L3 cache supported) by applications running
+on the platform. A Resource Monitoring ID (RMID) is the abstraction of the
+application(s) that will be monitored for its cache usage. The CMT hardware
+tracks cache utilization of memory accesses according to the RMID and reports
+monitored data via a counter register.
+
+For more detailed information please refer to Intel SDM chapter
+"17.14 - Platform Shared Resource Monitoring: Cache Monitoring Technology".
+
+In Xen's implementation, each domain in the system can be assigned a RMID
+independently, while RMID=0 is reserved for monitoring domains that don't
+have CMT service attached. RMID is opaque for xl/libxl and is only used in
+hypervisor.
+
+### xl interfaces
+
+A domain is assigned a RMID implicitly by attaching it to CMT service:
+
+`xl psr-cmt-attach <domid>`
+
+After that, cache usage for the domain can be shown by:
+
+`xl psr-cmt-show cache-occupancy <domid>`
+
+Once monitoring is not needed any more, the domain can be detached from the
+CMT service by:
+
+`xl psr-cmt-detach <domid>`
+
+An attach may fail because of no free RMID available. In such case unused
+RMID(s) can be freed by detaching corresponding domains from CMT service.
+
+Maximum RMID and supported monitor types in the system can be obtained by:
+
+`xl psr-cmt-hwinfo`
+
+## Memory Bandwidth Monitoring (MBM)
+
+Memory Bandwidth Monitoring(MBM) is a new hardware feature available on Intel
+Broadwell and later server platforms which builds on the CMT infrastructure to
+allow monitoring of system memory bandwidth. It introduces two new monitoring
+event type to monitor system total/local memory bandwidth. The same RMID can
+be used to monitor both cache usage and memory bandwidth at the same time.
+
+For more detailed information please refer to Intel SDM chapter
+"17.14 - Platform Shared Resource Monitoring: Cache Monitoring Technology".
+
+In Xen's implementation, MBM shares the same set of underlying monitoring
+service with CMT and can be used to monitor memory bandwidth on a per domain
+basis.
+
+The xl interfaces are the same with that of CMT. The difference is the
+monitor type is corresponding memory monitoring type (local-mem-bandwidth/
+total-mem-bandwidth instead of cache-occupancy). E.g. after a `xl psr-cmt-attach`:
+
+`xl psr-cmt-show local-mem-bandwidth <domid>`
+
+`xl psr-cmt-show total-mem-bandwidth <domid>`
+
+## Cache Allocation Technology (CAT)
+
+Cache Allocation Technology (CAT) is a new feature available on Intel
+Broadwell and later server platforms that allows an OS or Hypervisor/VMM to
+partition cache allocation (i.e. L3 cache) based on application priority or
+Class of Service (COS). Each COS is configured using capacity bitmasks (CBM)
+which represent cache capacity and indicate the degree of overlap and
+isolation between classes. System cache resource is divided into numbers of
+minimum portions which is then made up into subset for cache partition. Each
+portion corresponds to a bit in CBM and the set bit represents the
+corresponding cache portion is available.
+
+For example, assuming a system with 8 portions and 3 domains:
+
+ A CBM of 0xff for every domain means each domain can access the
+ whole cache. This is the default.
+
+ Giving one domain a CBM of 0x0f and the other two domain's 0xf0
+ means that the first domain gets exclusive access to half of the
+ cache (half of the portions) and the other two will share the
+ other half.
+
+ Giving one domain a CBM of 0x0f, one 0x30 and the last 0xc0
+ would give the first domain exclusive access to half the cache,
+ and the other two exclusive access to one quarter each.
+
+For more detailed information please refer to Intel SDM chapter
+"17.15 - Platform Shared Resource Control: Cache Allocation Technology".
+
+In Xen's implementation, CBM can be configured with libxl/xl interfaces but
+COS is maintained in hypervisor only. The cache partition granularity is per
+domain, each domain has COS=0 assigned by default, the corresponding CBM is
+all-ones, which means all the cache resource can be used by default.
+
+### xl interfaces
+
+System CAT information such as maximum COS and CBM length can be obtained by:
+
+`xl psr-cat-hwinfo`
+
+The simplest way to change a domain's CBM from its default is running:
+
+`xl psr-cat-cbm-set [OPTIONS] <domid> <cbm>`
+
+where cbm is a number to represent the corresponding cache subset can be used.
+A cbm is valid only when:
+
+ * Set bits only exist in the range of [0, cbm_len), where cbm_len can be
+ obtained with `xl psr-cat-hwinfo`.
+ * All the set bits are contiguous.
+
+In a multi-socket system, the same cbm will be set on each socket by default.
+Per socket cbm can be specified with the `--socket SOCKET` option.
+
+Setting the CBM may not be successful if insufficient COS is available. In
+such case unused COS(es) may be freed by setting CBM of all related domains to
+its default value(all-ones).
+
+Per domain CBM settings can be shown by:
+
+`xl psr-cat-show`
+
+## Reference
+
+[1] Intel SDM
+(http://www.intel.com/content/www/us/en/processors/architectures-software-developer-manuals.html).
--
1.9.1
^ permalink raw reply related [flat|nested] 33+ messages in thread* Re: [PATCH v5 13/13] docs: add xl-psr.markdown
2015-04-17 14:33 ` [PATCH v5 13/13] docs: add xl-psr.markdown Chao Peng
@ 2015-04-21 14:05 ` Ian Campbell
0 siblings, 0 replies; 33+ messages in thread
From: Ian Campbell @ 2015-04-21 14:05 UTC (permalink / raw)
To: Chao Peng
Cc: keir, stefano.stabellini, andrew.cooper3, Ian.Jackson, xen-devel,
will.auld, JBeulich, wei.liu2, dgdegra
On Fri, 2015-04-17 at 22:33 +0800, Chao Peng wrote:
> @@ -1534,7 +1540,8 @@ Show CAT hardware information.
>
> =item B<psr-cat-cbm-set> [I<OPTIONS>] I<domain-id> I<cbm>
>
> -Set cache capacity bitmasks(CBM) for a domain.
> +Set cache capacity bitmasks(CBM) for a domain. For how to specify I<cbm>
> +please refer to the link above.
I think the "link above" is to far. I think say something more explicit
like L<xl-psr.txt> ?
>
> B<OPTIONS>
>
> @@ -1575,6 +1582,7 @@ And the following documents on the xen.org website:
> L<http://xenbits.xen.org/docs/unstable/misc/xl-network-configuration.html>
> L<http://xenbits.xen.org/docs/unstable/misc/xl-disk-configuration.txt>
> L<http://xenbits.xen.org/docs/unstable/misc/xsm-flask.txt>
> +L<http://xenbits.xen.org/docs/unstable/misc/xl-psr.html>
>
> For systems that don't automatically bring CPU online:
>
> diff --git a/docs/misc/xl-psr.markdown b/docs/misc/xl-psr.markdown
> new file mode 100644
> index 0000000..d167b84
> --- /dev/null
> +++ b/docs/misc/xl-psr.markdown
> @@ -0,0 +1,134 @@
> +# Intel Platform Shared Resource Monitoring/Control in xl
> +
> +This document introduces Intel Platform Shared Resource Monitoring/Control
> +technologies, their basic concepts and the xl interfaces.
> +
> +## Cache Monitoring Technology (CMT)
> +
> +Cache Monitoring Technology (CMT) is a new feature available on Intel Haswell
> +and later server platforms that allows an OS or Hypervisor/VMM to determine
> +the usage of cache(currently only L3 cache supported) by applications running
^space before ( please.
> +For example, assuming a system with 8 portions and 3 domains:
> +
> + A CBM of 0xff for every domain means each domain can access the
> + whole cache. This is the default.
> +
> + Giving one domain a CBM of 0x0f and the other two domain's 0xf0
> + means that the first domain gets exclusive access to half of the
> + cache (half of the portions) and the other two will share the
> + other half.
> +
> + Giving one domain a CBM of 0x0f, one 0x30 and the last 0xc0
> + would give the first domain exclusive access to half the cache,
> + and the other two exclusive access to one quarter each.
How does markdown render this? I think you might want to start each para
with a * to make it a bullet list.
Other than those minor things all looks good, thanks.
Ian.
^ permalink raw reply [flat|nested] 33+ messages in thread