[Qemu-devel] [RFC PATCH 2/5] hw/i386: Add AMD EPYC topology encoding

["Moger, Babu" <Babu.Moger@amd.com>]

Currently, the apicid is a sequential number in x86 cpu models. This
works fine for most of the cases. But, in certain cases this will
result into cpu topology inconsistency. This problem was observed in
AMD EPYC cpu models.

To address that we need to build apicid as per the hardware specification.
We are attempting to build the topology close to the hardware as much as
possible.

Per Processor Programming Reference (PPR) for AMD Family 17h Models,
we need to follow the following decoding.

    2.1.10.2.1.3
    ApicId Enumeration Requirements
    Operating systems are expected to use
    Core::X86::Cpuid::SizeId[ApicIdCoreIdSize], the number of least
    significant bits in the Initial APIC ID that indicate core ID within a
    processor, in constructing per-core CPUID
    masks. Core::X86::Cpuid::SizeId[ApicIdCoreIdSize] determines the maximum
    number of cores (MNC) that the
    processor could theoretically support, not the actual number of cores that
    are actually implemented or enabled on
    the processor, as indicated by Core::X86::Cpuid::SizeId[NC].
    Each Core::X86::Apic::ApicId[ApicId] register is preset as follows:
    • ApicId[6] = Socket ID.
    • ApicId[5:4] = Node ID.
    • ApicId[3] = Logical CCX L3 complex ID
    • ApicId[2:0]= (SMT) ? {LogicalCoreID[1:0],ThreadId} :
    {1'b0,LogicalCoreID[1:0]}.

Add the structures and functions to decode the ApicId for AMD EPYC models.

Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1728166

Signed-off-by: Babu Moger <babu.moger@amd.com>
---
 include/hw/i386/topology.h | 140 +++++++++++++++++++++++++++++++++++++
 1 file changed, 140 insertions(+)

diff --git a/include/hw/i386/topology.h b/include/hw/i386/topology.h
index 4ff5b2da6c..0b2fd46b41 100644
--- a/include/hw/i386/topology.h
+++ b/include/hw/i386/topology.h
@@ -50,6 +50,7 @@ typedef struct X86CPUTopoInfo {
     unsigned die_id;
     unsigned core_id;
     unsigned smt_id;
+    unsigned ccx_id;
 } X86CPUTopoInfo;
 
 /* Return the bit width needed for 'count' IDs
@@ -179,4 +180,143 @@ static inline apic_id_t x86_apicid_from_cpu_idx(unsigned nr_dies,
     return apicid_from_topo_ids(nr_dies, nr_cores, nr_threads, &topo);
 }
 
+/* Definitions used for building CPUID Leaf 0x8000001D and 0x8000001E
+ * Please refer to the AMD64 Architecture Programmer’s Manual Volume 3.
+ * Define the constants to build the cpu topology. Right now, TOPOEXT
+ * feature is enabled only on EPYC. So, these constants are based on
+ * EPYC supported configurations. We may need to handle the cases if
+ * these values change in future.
+ */
+
+/* Maximum core complexes in a node */
+#define MAX_CCX                  2
+/* Maximum cores in a core complex */
+#define MAX_CORES_IN_CCX         4
+/* Maximum cores in a node */
+#define MAX_CORES_IN_DIE         8
+#define CCX_OFFSET_EPYC          3
+#define DIE_OFFSET_EPYC          4
+#define PKG_OFFSET_EPYC          6
+
+/* Figure out the number of dies required to build this config.
+ * Max cores in a die is 8
+ */
+static inline int dies_in_socket(int numa_nodes, int smp_sockets, int smp_cores)
+{
+    /*
+     * Create a config with user given (nr_nodes > 1) numa node config,
+     * else go with a standard configuration
+     */
+    if (numa_nodes > 1)
+        return DIV_ROUND_UP(numa_nodes, smp_sockets);
+    else
+        return DIV_ROUND_UP(smp_cores, MAX_CORES_IN_DIE);
+}
+
+/* Decide the number of cores in a core complex with the given nr_cores using
+ * following set constants MAX_CCX, MAX_CORES_IN_CCX, MAX_CORES_IN_DIE and
+ * MAX_NODES_PER_SOCKET. Maintain symmetry as much as possible
+ * L3 cache is shared across all cores in a core complex. So, this will also
+ * tell us how many cores are sharing the L3 cache.
+ */
+static inline int cores_in_ccx(int numa_nodes, int smp_sockets, int smp_cores)
+{
+    int dies;
+
+    /* Check if we can fit all the cores in one core complex */
+    if (smp_cores <= MAX_CORES_IN_CCX) {
+        return smp_cores;
+    }
+
+    /* Get the number of nodes required to build this config */
+    dies = dies_in_socket(numa_nodes, smp_sockets, smp_cores);
+
+    /*
+     * Divide the cores accros all the core complexes
+     * Return rounded up value
+     */
+    return DIV_ROUND_UP(smp_cores, dies * MAX_CCX);
+}
+
+/* Calculate thread/core/package IDs for a specific topology,
+ * based on APIC ID
+ */
+static inline void x86_topo_ids_from_apicid_epyc(apic_id_t apicid,
+                                                 unsigned smp_dies,
+                                                 unsigned smp_cores,
+                                                 unsigned smp_threads,
+                                                 X86CPUTopoInfo *topo)
+{
+    topo->smt_id = apicid &
+                   ~(0xFFFFFFFFUL << apicid_smt_width(smp_dies, smp_cores, smp_threads));
+    topo->core_id = (apicid >> apicid_core_offset(smp_dies, smp_cores, smp_threads)) &
+                   ~(0xFFFFFFFFUL << apicid_core_width(smp_dies, smp_cores, smp_threads));
+    topo->ccx_id = (apicid >> CCX_OFFSET_EPYC) & 1;
+    topo->die_id = (apicid >> DIE_OFFSET_EPYC) & 3;
+    topo->pkg_id = apicid >> PKG_OFFSET_EPYC;
+}
+
+/* Calculate thread/core/package IDs for a specific topology,
+ * based on (contiguous) CPU index
+ * Processor Programming Reference (PPR) for AMD Family 17h Model 01h,
+ * Revision B1 Processors
+ * ApicId Enumeration Requirements
+ • ApicId[6] = Socket ID.
+ • ApicId[5:4] = Node ID.
+ • ApicId[3] = Logical CCX L3 complex ID
+ • ApicId[2:0]= (SMT) ? {LogicalCoreID[1:0],ThreadId} : {1'b0,LogicalCoreID[1:0]}.
+ */
+static inline void x86_topo_ids_from_idx_epyc(unsigned numa_nodes,
+		                              unsigned smp_sockets,
+					      unsigned smp_cores,
+                                              unsigned smp_threads,
+                                              unsigned cpu_index,
+                                              X86CPUTopoInfo *topo)
+{
+    unsigned core_index = cpu_index / smp_threads;
+    unsigned core_id = core_index % smp_cores; /* Core id inside the socket */
+    unsigned ccx_cores;
+
+    topo->smt_id = cpu_index % smp_threads;
+
+    ccx_cores = cores_in_ccx(numa_nodes, smp_sockets, smp_cores);
+
+    topo->core_id = core_id % ccx_cores; /* core id inside the ccx */
+    topo->ccx_id = (core_id % (ccx_cores * MAX_CCX)) / ccx_cores;
+    topo->die_id = core_id / (ccx_cores * MAX_CCX);
+    topo->pkg_id = core_index / smp_cores;
+}
+
+/* Make APIC ID for the CPU based on Pkg_ID, Core_ID, SMT_ID
+ *
+ * The caller must make sure core_id < nr_cores and smt_id < nr_threads.
+ */
+static inline apic_id_t apicid_from_topo_ids_epyc(unsigned smp_cores,
+                                                  unsigned smp_threads,
+                                                  const X86CPUTopoInfo *topo)
+{
+    if (smp_threads - 1)
+        return ((topo->pkg_id << PKG_OFFSET_EPYC) | (topo->die_id << DIE_OFFSET_EPYC) |
+                (topo->ccx_id << CCX_OFFSET_EPYC) | (topo->core_id << 1) | (topo->smt_id));
+    else
+        return ((topo->pkg_id << PKG_OFFSET_EPYC) | (topo->die_id << DIE_OFFSET_EPYC) |
+                (topo->ccx_id << CCX_OFFSET_EPYC) | (topo->core_id));
+}
+
+/* Make APIC ID for the CPU 'cpu_index'
+ *
+ * 'cpu_index' is a sequential, contiguous ID for the CPU.
+ */
+static inline apic_id_t x86_apicid_from_cpu_idx_epyc(unsigned numa_nodes,
+		                                     unsigned smp_sockets,
+                                                     unsigned smp_cores,
+                                                     unsigned smp_threads,
+                                                     unsigned cpu_index)
+{
+    X86CPUTopoInfo topo;
+    x86_topo_ids_from_idx_epyc(numa_nodes, smp_sockets, smp_cores, smp_threads,
+		               cpu_index, &topo);
+    return apicid_from_topo_ids_epyc(smp_cores, smp_threads, &topo);
+}
+
 #endif /* HW_I386_TOPOLOGY_H */
-- 
2.20.1