From: Ravi Bangoria <ravi.bangoria@amd.com>
To: <peterz@infradead.org>, <acme@kernel.org>
Cc: <ravi.bangoria@amd.com>, <jolsa@kernel.org>,
<namhyung@kernel.org>, <eranian@google.com>, <irogers@google.com>,
<jmario@redhat.com>, <leo.yan@linaro.org>, <alisaidi@amazon.com>,
<ak@linux.intel.com>, <kan.liang@linux.intel.com>,
<dave.hansen@linux.intel.com>, <hpa@zytor.com>,
<mingo@redhat.com>, <mark.rutland@arm.com>,
<alexander.shishkin@linux.intel.com>, <tglx@linutronix.de>,
<bp@alien8.de>, <x86@kernel.org>,
<linux-perf-users@vger.kernel.org>,
<linux-kernel@vger.kernel.org>, <sandipan.das@amd.com>,
<ananth.narayan@amd.com>, <kim.phillips@amd.com>,
<santosh.shukla@amd.com>
Subject: [PATCH 03/13] perf/x86/amd: Support PERF_SAMPLE_DATA_SRC based on IBS_OP_DATA*
Date: Wed, 25 May 2022 15:09:28 +0530 [thread overview]
Message-ID: <20220525093938.4101-4-ravi.bangoria@amd.com> (raw)
In-Reply-To: <20220525093938.4101-1-ravi.bangoria@amd.com>
struct perf_mem_data_src is used to pass arch specific memory access
details into generic form. These details gets consumed by tools like
perf mem and c2c. Each IBS tagged load/store sample provides most of
the information needed for these tools. Add a logic to convert IBS
specific raw data into perf_mem_data_src.
Signed-off-by: Ravi Bangoria <ravi.bangoria@amd.com>
---
arch/x86/events/amd/ibs.c | 297 +++++++++++++++++++++++++++++++++++++-
1 file changed, 291 insertions(+), 6 deletions(-)
diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c
index c251bc44c088..6626caeed6a1 100644
--- a/arch/x86/events/amd/ibs.c
+++ b/arch/x86/events/amd/ibs.c
@@ -688,6 +688,289 @@ static struct perf_ibs perf_ibs_op = {
.get_count = get_ibs_op_count,
};
+static void perf_ibs_get_mem_op(u64 op_data3, struct perf_sample_data *data)
+{
+ union perf_mem_data_src *data_src = &data->data_src;
+
+ data_src->mem_op = PERF_MEM_OP_NA;
+
+ if (op_data3 & IBS_LD_OP_MASK)
+ data_src->mem_op = PERF_MEM_OP_LOAD;
+ else if (op_data3 & IBS_ST_OP_MASK)
+ data_src->mem_op = PERF_MEM_OP_STORE;
+}
+
+/*
+ * Processors having CPUID_Fn8000001B_EAX[11] aka IBS_CAPS_ZEN4 has
+ * more fine granular DataSrc encodings. Others have coarse.
+ */
+static u8 perf_ibs_data_src(u64 op_data2)
+{
+ if (ibs_caps & IBS_CAPS_ZEN4) {
+ return ((op_data2 & IBS_DATA_SRC_HI_MASK) >> (IBS_DATA_SRC_HI_SHIFT - 3)) |
+ ((op_data2 & IBS_DATA_SRC_LO_MASK) >> IBS_DATA_SRC_LO_SHIFT);
+ }
+
+ return (op_data2 & IBS_DATA_SRC_LO_MASK) >> IBS_DATA_SRC_LO_SHIFT;
+}
+
+static void perf_ibs_get_mem_lvl(struct perf_event *event, u64 op_data2,
+ u64 op_data3, struct perf_sample_data *data)
+{
+ union perf_mem_data_src *data_src = &data->data_src;
+ u8 ibs_data_src = perf_ibs_data_src(op_data2);
+
+ data_src->mem_lvl = 0;
+
+ /*
+ * DcMiss, L2Miss, DataSrc, DcMissLat etc. are all invalid for Uncached
+ * memory accesses. So, check DcUcMemAcc bit early.
+ */
+ if (op_data3 & IBS_DC_UC_MEM_ACC_MASK &&
+ ibs_data_src != IBS_DATA_SRC_EXT_IO) {
+ data_src->mem_lvl = PERF_MEM_LVL_UNC | PERF_MEM_LVL_HIT;
+ return;
+ }
+
+ /* L1 Hit */
+ if ((op_data3 & IBS_DC_MISS_MASK) == 0) {
+ data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT;
+ return;
+ }
+
+ /* L2 Hit */
+ if ((op_data3 & IBS_L2_MISS_MASK) == 0) {
+ /* Erratum #1293 */
+ if (boot_cpu_data.x86 != 0x19 || boot_cpu_data.x86_model > 0xF ||
+ !(op_data3 & IBS_SW_PF_MASK || op_data3 & IBS_DC_MISS_NO_MAB_ALLOC_MASK)) {
+ data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT;
+ return;
+ }
+ }
+
+ /* L3 Hit */
+ if (ibs_caps & IBS_CAPS_ZEN4) {
+ if (data_src->mem_op == PERF_MEM_OP_LOAD &&
+ ibs_data_src == IBS_DATA_SRC_EXT_LOC_CACHE) {
+ data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT;
+ return;
+ }
+ } else {
+ if (data_src->mem_op == PERF_MEM_OP_LOAD &&
+ ibs_data_src == IBS_DATA_SRC_LOC_CACHE) {
+ data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_REM_CCE1 |
+ PERF_MEM_LVL_HIT;
+ return;
+ }
+ }
+
+ /* A peer cache in a near CCX. */
+ if (ibs_caps & IBS_CAPS_ZEN4 && data_src->mem_op == PERF_MEM_OP_LOAD &&
+ ibs_data_src == IBS_DATA_SRC_EXT_NEAR_CCX_CACHE) {
+ data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1 | PERF_MEM_LVL_HIT;
+ return;
+ }
+
+ /* A peer cache in a far CCX. */
+ if (ibs_caps & IBS_CAPS_ZEN4) {
+ if (data_src->mem_op == PERF_MEM_OP_LOAD &&
+ ibs_data_src == IBS_DATA_SRC_EXT_FAR_CCX_CACHE) {
+ data_src->mem_lvl = PERF_MEM_LVL_REM_CCE2 | PERF_MEM_LVL_HIT;
+ return;
+ }
+ } else {
+ if (data_src->mem_op == PERF_MEM_OP_LOAD &&
+ ibs_data_src == IBS_DATA_SRC_REM_CACHE) {
+ data_src->mem_lvl = PERF_MEM_LVL_REM_CCE2 | PERF_MEM_LVL_HIT;
+ return;
+ }
+ }
+
+ /* DRAM */
+ if (data_src->mem_op == PERF_MEM_OP_LOAD &&
+ ibs_data_src == IBS_DATA_SRC_EXT_DRAM) {
+ if ((op_data2 & IBS_RMT_NODE_MASK) == 0)
+ data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT;
+ else
+ data_src->mem_lvl = PERF_MEM_LVL_REM_RAM1 | PERF_MEM_LVL_HIT;
+ return;
+ }
+
+ /* PMEM */
+ if (ibs_caps & IBS_CAPS_ZEN4 && data_src->mem_op == PERF_MEM_OP_LOAD &&
+ ibs_data_src == IBS_DATA_SRC_EXT_PMEM) {
+ data_src->mem_lvl_num = PERF_MEM_LVLNUM_PMEM;
+ if (op_data2 & IBS_RMT_NODE_MASK) {
+ data_src->mem_remote = PERF_MEM_REMOTE_REMOTE;
+ /* IBS doesn't provide Remote socket detail */
+ data_src->mem_hops = PERF_MEM_HOPS_1;
+ }
+ return;
+ }
+
+ /* Extension Memory */
+ if (ibs_caps & IBS_CAPS_ZEN4 && data_src->mem_op == PERF_MEM_OP_LOAD &&
+ ibs_data_src == IBS_DATA_SRC_EXT_EXT_MEM) {
+ data_src->mem_lvl_num = PERF_MEM_LVLNUM_EXTN_MEM;
+ if (op_data2 & IBS_RMT_NODE_MASK) {
+ data_src->mem_remote = PERF_MEM_REMOTE_REMOTE;
+ /* IBS doesn't provide Remote socket detail */
+ data_src->mem_hops = PERF_MEM_HOPS_1;
+ }
+ return;
+ }
+
+ /* IO */
+ if (data_src->mem_op == PERF_MEM_OP_LOAD &&
+ ibs_data_src == IBS_DATA_SRC_EXT_IO) {
+ data_src->mem_lvl_num = PERF_MEM_LVLNUM_IO;
+ if (op_data2 & IBS_RMT_NODE_MASK) {
+ data_src->mem_remote = PERF_MEM_REMOTE_REMOTE;
+ /* IBS doesn't provide Remote socket detail */
+ data_src->mem_hops = PERF_MEM_HOPS_1;
+ }
+ return;
+ }
+
+ /*
+ * MAB (Miss Address Buffer) Hit. MAB keeps track of outstanding
+ * DC misses. However such data may come from any level in mem
+ * hierarchy. IBS provides detail about both MAB as well as actual
+ * DataSrc simultaneously. Prioritize DataSrc over MAB, i.e. set
+ * MAB only when IBS fails to provide DataSrc.
+ */
+ if (op_data3 & IBS_DC_MISS_NO_MAB_ALLOC_MASK) {
+ data_src->mem_lvl = PERF_MEM_LVL_LFB | PERF_MEM_LVL_HIT;
+ return;
+ }
+
+ data_src->mem_lvl = PERF_MEM_LVL_NA;
+}
+
+static bool perf_ibs_cache_hit_st_valid(void)
+{
+ /* 0: Uninitialized, 1: Valid, -1: Invalid */
+ static int cache_hist_st_valid;
+
+ if (unlikely(!cache_hist_st_valid)) {
+ if (boot_cpu_data.x86 == 0x19 &&
+ (boot_cpu_data.x86_model <= 0xF ||
+ (boot_cpu_data.x86_model >= 0x20 &&
+ boot_cpu_data.x86_model <= 0x5F))) {
+ cache_hist_st_valid = -1;
+ } else {
+ cache_hist_st_valid = 1;
+ }
+ }
+
+ return cache_hist_st_valid == 1;
+}
+
+static void perf_ibs_get_mem_snoop(u64 op_data2, struct perf_sample_data *data)
+{
+ union perf_mem_data_src *data_src = &data->data_src;
+ u8 ibs_data_src;
+
+ data_src->mem_snoop = PERF_MEM_SNOOP_NA;
+
+ if (!perf_ibs_cache_hit_st_valid() ||
+ data_src->mem_op != PERF_MEM_OP_LOAD ||
+ data_src->mem_lvl & PERF_MEM_LVL_L1 ||
+ data_src->mem_lvl & PERF_MEM_LVL_L2 ||
+ op_data2 & IBS_CACHE_HIT_ST_MASK)
+ return;
+
+ ibs_data_src = perf_ibs_data_src(op_data2);
+
+ if ((ibs_data_src == IBS_DATA_SRC_LOC_CACHE) ||
+ (ibs_caps & IBS_CAPS_ZEN4 && (
+ ibs_data_src == IBS_DATA_SRC_EXT_LOC_CACHE ||
+ ibs_data_src == IBS_DATA_SRC_EXT_NEAR_CCX_CACHE ||
+ ibs_data_src == IBS_DATA_SRC_EXT_FAR_CCX_CACHE))) {
+ data_src->mem_snoop = PERF_MEM_SNOOP_HITM;
+ }
+}
+
+static void perf_ibs_get_tlb_lvl(u64 op_data3, struct perf_sample_data *data)
+{
+ union perf_mem_data_src *data_src = &data->data_src;
+ u64 l1_tlb_miss = op_data3 & IBS_DC_L1_TLB_MISS_MASK;
+ u64 lin_addr_valid = op_data3 & IBS_DC_LIN_ADDR_VALID_MASK;
+ u64 l2_tlb_miss = op_data3 & IBS_DC_L2_TLB_MISS_MASK;
+
+ data_src->mem_dtlb = PERF_MEM_TLB_NA;
+
+ if (!lin_addr_valid)
+ return;
+
+ if (!l1_tlb_miss) {
+ data_src->mem_dtlb = PERF_MEM_TLB_L1 | PERF_MEM_TLB_HIT;
+ return;
+ }
+
+ if (!l2_tlb_miss) {
+ data_src->mem_dtlb = PERF_MEM_TLB_L2 | PERF_MEM_TLB_HIT;
+ return;
+ }
+
+ data_src->mem_dtlb = PERF_MEM_TLB_L2 | PERF_MEM_TLB_MISS;
+}
+
+static void perf_ibs_get_mem_lock(u64 op_data3, struct perf_sample_data *data)
+{
+ union perf_mem_data_src *data_src = &data->data_src;
+
+ data_src->mem_lock = PERF_MEM_LOCK_NA;
+
+ if (op_data3 & IBS_DC_LOCKED_OP_MASK)
+ data_src->mem_lock = PERF_MEM_LOCK_LOCKED;
+}
+
+#define ibs_op_msr_idx(msr) (msr - MSR_AMD64_IBSOPCTL)
+
+static void perf_ibs_get_data_src(struct perf_event *event,
+ struct perf_ibs_data *ibs_data,
+ struct perf_sample_data *data)
+{
+ union perf_mem_data_src *data_src = &data->data_src;
+ u64 op_data2 = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA2)];
+ u64 op_data3 = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)];
+
+ perf_ibs_get_mem_op(op_data3, data);
+ if (data_src->mem_op != PERF_MEM_OP_LOAD &&
+ data_src->mem_op != PERF_MEM_OP_STORE)
+ return;
+
+ /* Erratum #1293 */
+ if (boot_cpu_data.x86 == 0x19 && boot_cpu_data.x86_model <= 0xF &&
+ (op_data3 & IBS_SW_PF_MASK ||
+ op_data3 & IBS_DC_MISS_NO_MAB_ALLOC_MASK)) {
+ /*
+ * OP_DATA2 has only two fields on Zen3: DataSrc and RmtNode.
+ * DataSrc=0 is No valid status and RmtNode is invalid when
+ * DataSrc=0.
+ */
+ op_data2 = 0;
+ }
+
+ perf_ibs_get_mem_lvl(event, op_data2, op_data3, data);
+ perf_ibs_get_mem_snoop(op_data2, data);
+ perf_ibs_get_tlb_lvl(op_data3, data);
+ perf_ibs_get_mem_lock(op_data3, data);
+}
+
+static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, u64 sample_type,
+ int check_rip)
+{
+ if (sample_type & PERF_SAMPLE_RAW ||
+ (perf_ibs == &perf_ibs_op &&
+ sample_type & PERF_SAMPLE_DATA_SRC))
+ return perf_ibs->offset_max;
+ else if (check_rip)
+ return 3;
+ return 1;
+}
+
static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
{
struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
@@ -735,12 +1018,9 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
size = 1;
offset = 1;
check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK));
- if (event->attr.sample_type & PERF_SAMPLE_RAW)
- offset_max = perf_ibs->offset_max;
- else if (check_rip)
- offset_max = 3;
- else
- offset_max = 1;
+
+ offset_max = perf_ibs_get_offset_max(perf_ibs, event->attr.sample_type, check_rip);
+
do {
rdmsrl(msr + offset, *buf++);
size++;
@@ -793,6 +1073,11 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
data.raw = &raw;
}
+ if (perf_ibs == &perf_ibs_op) {
+ if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC)
+ perf_ibs_get_data_src(event, &ibs_data, &data);
+ }
+
/*
* rip recorded by IbsOpRip will not be consistent with rsp and rbp
* recorded as part of interrupt regs. Thus we need to use rip from
--
2.31.1
next prev parent reply other threads:[~2022-05-25 9:42 UTC|newest]
Thread overview: 23+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-05-25 9:39 [PATCH 00/13] perf mem/c2c: Add support for AMD Ravi Bangoria
2022-05-25 9:39 ` [PATCH 01/13] perf/mem: Introduce PERF_MEM_LVLNUM_{EXTN_MEM|IO} Ravi Bangoria
2022-05-25 9:39 ` [PATCH 02/13] perf/x86/amd: Add IBS OP_DATA2/3 register bit definitions Ravi Bangoria
2022-05-26 15:08 ` Kim Phillips
2022-06-01 4:25 ` Ravi Bangoria
2022-05-25 9:39 ` Ravi Bangoria [this message]
2022-05-25 9:39 ` [PATCH 04/13] perf/x86/amd: Support PERF_SAMPLE_WEIGHT using IBS OP_DATA3[IbsDcMissLat] Ravi Bangoria
2022-05-25 12:58 ` Stephane Eranian
2022-05-26 12:14 ` Ravi Bangoria
2022-05-25 9:39 ` [PATCH 05/13] perf/x86/amd: Support PERF_SAMPLE_ADDR using IBS_DC_LINADDR Ravi Bangoria
2022-05-25 9:39 ` [PATCH 06/13] perf/x86/amd: Support PERF_SAMPLE_PHY_ADDR using IBS_DC_PHYSADDR Ravi Bangoria
2022-05-25 11:21 ` Peter Zijlstra
2022-05-26 8:46 ` Ravi Bangoria
2022-05-26 9:56 ` Peter Zijlstra
2022-05-26 10:59 ` Ravi Bangoria
2022-05-26 11:09 ` Peter Zijlstra
2022-05-25 9:39 ` [PATCH 07/13] perf tool: Sync include/uapi/linux/perf_event.h header Ravi Bangoria
2022-05-25 9:39 ` [PATCH 08/13] perf tool: Sync arch/x86/include/asm/amd-ibs.h header Ravi Bangoria
2022-05-25 9:39 ` [PATCH 09/13] perf mem: Add support for printing PERF_MEM_LVLNUM_{EXTN_MEM|IO} Ravi Bangoria
2022-05-25 9:39 ` [PATCH 10/13] perf mem/c2c: Set PERF_SAMPLE_WEIGHT for LOAD_STORE events Ravi Bangoria
2022-05-25 9:39 ` [PATCH 11/13] perf mem/c2c: Add load store event mappings for AMD Ravi Bangoria
2022-05-25 9:39 ` [PATCH 12/13] perf mem/c2c: Avoid printing empty lines for unsupported events Ravi Bangoria
2022-05-25 9:39 ` [PATCH 13/13] perf mem: Use more generic term for LFB Ravi Bangoria
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220525093938.4101-4-ravi.bangoria@amd.com \
--to=ravi.bangoria@amd.com \
--cc=acme@kernel.org \
--cc=ak@linux.intel.com \
--cc=alexander.shishkin@linux.intel.com \
--cc=alisaidi@amazon.com \
--cc=ananth.narayan@amd.com \
--cc=bp@alien8.de \
--cc=dave.hansen@linux.intel.com \
--cc=eranian@google.com \
--cc=hpa@zytor.com \
--cc=irogers@google.com \
--cc=jmario@redhat.com \
--cc=jolsa@kernel.org \
--cc=kan.liang@linux.intel.com \
--cc=kim.phillips@amd.com \
--cc=leo.yan@linaro.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-perf-users@vger.kernel.org \
--cc=mark.rutland@arm.com \
--cc=mingo@redhat.com \
--cc=namhyung@kernel.org \
--cc=peterz@infradead.org \
--cc=sandipan.das@amd.com \
--cc=santosh.shukla@amd.com \
--cc=tglx@linutronix.de \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.