[PATCH v2 06/14] perf/x86/amd: Support PERF_SAMPLE_PHY_ADDR

[Ravi Bangoria <ravi.bangoria@amd.com>]

IBS_DC_PHYSADDR provides the physical data address for the tagged load/
store operation. Populate perf sample physical address using it.
Currently, physical address is unconditionally overwritten by generic
perf driver. Introduce internal only __PERF_SAMPLE_PHYS_ADDR_EARLY type
to notify generic code that arch pmu has already set physical address.

Signed-off-by: Ravi Bangoria <ravi.bangoria@amd.com>
---
 arch/x86/events/amd/ibs.c       | 34 ++++++++++++++++++++++++++++++++-
 include/uapi/linux/perf_event.h |  1 +
 kernel/events/core.c            |  4 +++-
 3 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c
index 9b3e265a9fed..d224abddc3af 100644
--- a/arch/x86/events/amd/ibs.c
+++ b/arch/x86/events/amd/ibs.c
@@ -310,6 +310,13 @@ static int perf_ibs_init(struct perf_event *event)
 	if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
 		event->attr.sample_type |= __PERF_SAMPLE_CALLCHAIN_EARLY;
 
+	/*
+	 * Setting _EARLY flag makes sure generic perf driver does not
+	 * overwrite physical address set by arch specific pmu driver.
+	 */
+	if (event->attr.sample_type & PERF_SAMPLE_PHYS_ADDR)
+		event->attr.sample_type |= __PERF_SAMPLE_PHYS_ADDR_EARLY;
+
 	return 0;
 }
 
@@ -999,13 +1006,36 @@ static void perf_ibs_get_data_addr(struct perf_event *event,
 	data->addr = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCLINAD)];
 }
 
+static void perf_ibs_get_phy_addr(struct perf_event *event,
+				  struct perf_ibs_data *ibs_data,
+				  struct perf_sample_data *data)
+{
+	union perf_mem_data_src *data_src = &data->data_src;
+	union ibs_op_data3 op_data3;
+
+	op_data3.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)];
+
+	if (!(event->attr.sample_type & PERF_SAMPLE_DATA_SRC))
+		perf_ibs_get_mem_op(&op_data3, data);
+
+	if ((data_src->mem_op != PERF_MEM_OP_LOAD &&
+	    data_src->mem_op != PERF_MEM_OP_STORE) ||
+	    !op_data3.dc_phy_addr_valid) {
+		data->phys_addr = 0x0;
+		return;
+	}
+
+	data->phys_addr = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSDCPHYSAD)];
+}
+
 static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, u64 sample_type,
 				   int check_rip)
 {
 	if (sample_type & PERF_SAMPLE_RAW ||
 	    (perf_ibs == &perf_ibs_op &&
 	    (sample_type & PERF_SAMPLE_DATA_SRC ||
-	     sample_type & PERF_SAMPLE_ADDR)))
+	     sample_type & PERF_SAMPLE_ADDR ||
+	     sample_type & PERF_SAMPLE_PHYS_ADDR)))
 		return perf_ibs->offset_max;
 	else if (check_rip)
 		return 3;
@@ -1119,6 +1149,8 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
 			perf_ibs_get_data_src(event, &ibs_data, &data);
 		if (event->attr.sample_type & PERF_SAMPLE_ADDR)
 			perf_ibs_get_data_addr(event, &ibs_data, &data);
+		if (event->attr.sample_type & PERF_SAMPLE_PHYS_ADDR)
+			perf_ibs_get_phy_addr(event, &ibs_data, &data);
 	}
 
 	/*
diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
index 1c3157c1be9d..daf7c337e53e 100644
--- a/include/uapi/linux/perf_event.h
+++ b/include/uapi/linux/perf_event.h
@@ -165,6 +165,7 @@ enum perf_event_sample_format {
 
 	PERF_SAMPLE_MAX = 1U << 25,		/* non-ABI */
 
+	__PERF_SAMPLE_PHYS_ADDR_EARLY		= 1ULL << 62, /* non-ABI; internal use */
 	__PERF_SAMPLE_CALLCHAIN_EARLY		= 1ULL << 63, /* non-ABI; internal use */
 };
 
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 80782cddb1da..f1b486410d0b 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -7403,8 +7403,10 @@ void perf_prepare_sample(struct perf_event_header *header,
 		header->size += size;
 	}
 
-	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
+	if (sample_type & PERF_SAMPLE_PHYS_ADDR &&
+	    !(sample_type & __PERF_SAMPLE_PHYS_ADDR_EARLY)) {
 		data->phys_addr = perf_virt_to_phys(data->addr);
+	}
 
 #ifdef CONFIG_CGROUP_PERF
 	if (sample_type & PERF_SAMPLE_CGROUP) {
-- 
2.31.1