Stable Archive on lore.kernel.org
 help / color / Atom feed
From: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
To: linux-kernel@vger.kernel.org
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
	stable@vger.kernel.org, aneesh.kumar@linux.ibm.com,
	mpe@ellerman.id.au, linuxppc-dev@lists.ozlabs.org,
	"Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>,
	Sandipan Das <sandipan@linux.ibm.com>
Subject: [PATCH 4.14 57/62] powerpc/mm: Fixup tlbie vs store ordering issue on POWER9
Date: Fri,  8 Nov 2019 19:50:45 +0100
Message-ID: <20191108174759.879781213@linuxfoundation.org> (raw)
In-Reply-To: <20191108174719.228826381@linuxfoundation.org>

From: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>

commit a5d4b5891c2f1f865a2def1eb0030f534e77ff86 upstream.

On POWER9, under some circumstances, a broadcast TLB invalidation
might complete before all previous stores have drained, potentially
allowing stale stores from becoming visible after the invalidation.
This works around it by doubling up those TLB invalidations which was
verified by HW to be sufficient to close the risk window.

This will be documented in a yet-to-be-published errata.

Cc: stable@vger.kernel.org # v4.14
Fixes: 1a472c9dba6b ("powerpc/mm/radix: Add tlbflush routines")
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
[mpe: Enable the feature in the DT CPU features code for all Power9,
      rename the feature to CPU_FTR_P9_TLBIE_BUG per benh.]
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20180323045627.16800-3-aneesh.kumar@linux.vnet.ibm.com/
[sandipan: Backported to v4.14]
Signed-off-by: Sandipan Das <sandipan@linux.ibm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
---
 arch/powerpc/include/asm/cputable.h    |    4 ++-
 arch/powerpc/kernel/dt_cpu_ftrs.c      |    3 ++
 arch/powerpc/kvm/book3s_64_mmu_radix.c |    3 ++
 arch/powerpc/kvm/book3s_hv_rm_mmu.c    |   11 ++++++++
 arch/powerpc/mm/hash_native_64.c       |   16 ++++++++++++
 arch/powerpc/mm/pgtable_64.c           |    1 
 arch/powerpc/mm/tlb-radix.c            |   41 ++++++++++++++++++++++++---------
 7 files changed, 66 insertions(+), 13 deletions(-)

--- a/arch/powerpc/include/asm/cputable.h
+++ b/arch/powerpc/include/asm/cputable.h
@@ -215,6 +215,7 @@ enum {
 #define CPU_FTR_DAWR			LONG_ASM_CONST(0x0400000000000000)
 #define CPU_FTR_DABRX			LONG_ASM_CONST(0x0800000000000000)
 #define CPU_FTR_PMAO_BUG		LONG_ASM_CONST(0x1000000000000000)
+#define CPU_FTR_P9_TLBIE_BUG		LONG_ASM_CONST(0x2000000000000000)
 #define CPU_FTR_POWER9_DD1		LONG_ASM_CONST(0x4000000000000000)
 
 #ifndef __ASSEMBLY__
@@ -475,7 +476,8 @@ enum {
 	    CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \
 	    CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \
 	    CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_DAWR | \
-	    CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_ARCH_300)
+	    CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | \
+	    CPU_FTR_P9_TLBIE_BUG)
 #define CPU_FTRS_POWER9_DD1 ((CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD1) & \
 			     (~CPU_FTR_SAO))
 #define CPU_FTRS_CELL	(CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \
--- a/arch/powerpc/kernel/dt_cpu_ftrs.c
+++ b/arch/powerpc/kernel/dt_cpu_ftrs.c
@@ -742,6 +742,9 @@ static __init void cpufeatures_cpu_quirk
 	 */
 	if ((version & 0xffffff00) == 0x004e0100)
 		cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD1;
+
+	if ((version & 0xffff0000) == 0x004e0000)
+		cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_BUG;
 }
 
 static void __init cpufeatures_setup_finished(void)
--- a/arch/powerpc/kvm/book3s_64_mmu_radix.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c
@@ -160,6 +160,9 @@ static void kvmppc_radix_tlbie_page(stru
 	asm volatile("ptesync": : :"memory");
 	asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
 		     : : "r" (addr), "r" (kvm->arch.lpid) : "memory");
+	if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG))
+		asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
+			     : : "r" (addr), "r" (kvm->arch.lpid) : "memory");
 	asm volatile("ptesync": : :"memory");
 }
 
--- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c
+++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
@@ -448,6 +448,17 @@ static void do_tlbies(struct kvm *kvm, u
 			asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : :
 				     "r" (rbvalues[i]), "r" (kvm->arch.lpid));
 		}
+
+		if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
+			/*
+			 * Need the extra ptesync to make sure we don't
+			 * re-order the tlbie
+			 */
+			asm volatile("ptesync": : :"memory");
+			asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : :
+				     "r" (rbvalues[0]), "r" (kvm->arch.lpid));
+		}
+
 		asm volatile("eieio; tlbsync; ptesync" : : : "memory");
 		kvm->arch.tlbie_lock = 0;
 	} else {
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -104,6 +104,15 @@ static inline unsigned long  ___tlbie(un
 	return va;
 }
 
+static inline void fixup_tlbie(unsigned long vpn, int psize, int apsize, int ssize)
+{
+	if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
+		/* Need the extra ptesync to ensure we don't reorder tlbie*/
+		asm volatile("ptesync": : :"memory");
+		___tlbie(vpn, psize, apsize, ssize);
+	}
+}
+
 static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
 {
 	unsigned long rb;
@@ -181,6 +190,7 @@ static inline void tlbie(unsigned long v
 		asm volatile("ptesync": : :"memory");
 	} else {
 		__tlbie(vpn, psize, apsize, ssize);
+		fixup_tlbie(vpn, psize, apsize, ssize);
 		asm volatile("eieio; tlbsync; ptesync": : :"memory");
 	}
 	if (lock_tlbie && !use_local)
@@ -674,7 +684,7 @@ static void native_hpte_clear(void)
  */
 static void native_flush_hash_range(unsigned long number, int local)
 {
-	unsigned long vpn;
+	unsigned long vpn = 0;
 	unsigned long hash, index, hidx, shift, slot;
 	struct hash_pte *hptep;
 	unsigned long hpte_v;
@@ -746,6 +756,10 @@ static void native_flush_hash_range(unsi
 				__tlbie(vpn, psize, psize, ssize);
 			} pte_iterate_hashed_end();
 		}
+		/*
+		 * Just do one more with the last used values.
+		 */
+		fixup_tlbie(vpn, psize, psize, ssize);
 		asm volatile("eieio; tlbsync; ptesync":::"memory");
 
 		if (lock_tlbie)
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -491,6 +491,7 @@ void mmu_partition_table_set_entry(unsig
 			     "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
 		trace_tlbie(lpid, 0, TLBIEL_INVAL_SET_LPID, lpid, 2, 0, 0);
 	}
+	/* do we need fixup here ?*/
 	asm volatile("eieio; tlbsync; ptesync" : : : "memory");
 }
 EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry);
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -23,6 +23,33 @@
 #define RIC_FLUSH_PWC 1
 #define RIC_FLUSH_ALL 2
 
+static inline void __tlbie_va(unsigned long va, unsigned long pid,
+			      unsigned long ap, unsigned long ric)
+{
+	unsigned long rb,rs,prs,r;
+
+	rb = va & ~(PPC_BITMASK(52, 63));
+	rb |= ap << PPC_BITLSHIFT(58);
+	rs = pid << PPC_BITLSHIFT(31);
+	prs = 1; /* process scoped */
+	r = 1;   /* raidx format */
+
+	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
+		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+	trace_tlbie(0, 0, rb, rs, ric, prs, r);
+}
+
+static inline void fixup_tlbie(void)
+{
+	unsigned long pid = 0;
+	unsigned long va = ((1UL << 52) - 1);
+
+	if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
+		asm volatile("ptesync": : :"memory");
+		__tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
+	}
+}
+
 static inline void __tlbiel_pid(unsigned long pid, int set,
 				unsigned long ric)
 {
@@ -80,6 +107,7 @@ static inline void _tlbie_pid(unsigned l
 	asm volatile("ptesync": : :"memory");
 	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
 		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+	fixup_tlbie();
 	asm volatile("eieio; tlbsync; ptesync": : :"memory");
 	trace_tlbie(0, 0, rb, rs, ric, prs, r);
 }
@@ -105,19 +133,10 @@ static inline void _tlbiel_va(unsigned l
 static inline void _tlbie_va(unsigned long va, unsigned long pid,
 			     unsigned long ap, unsigned long ric)
 {
-	unsigned long rb,rs,prs,r;
-
-	rb = va & ~(PPC_BITMASK(52, 63));
-	rb |= ap << PPC_BITLSHIFT(58);
-	rs = pid << PPC_BITLSHIFT(31);
-	prs = 1; /* process scoped */
-	r = 1;   /* raidx format */
-
 	asm volatile("ptesync": : :"memory");
-	asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
-		     : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+	__tlbie_va(va, pid, ap, ric);
+	fixup_tlbie();
 	asm volatile("eieio; tlbsync; ptesync": : :"memory");
-	trace_tlbie(0, 0, rb, rs, ric, prs, r);
 }
 
 /*



  parent reply index

Thread overview: 66+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2019-11-08 18:49 [PATCH 4.14 00/62] 4.14.153-stable review Greg Kroah-Hartman
2019-11-08 18:49 ` [PATCH 4.14 01/62] arm64: dts: Fix gpio to pinmux mapping Greg Kroah-Hartman
2019-11-08 18:49 ` [PATCH 4.14 02/62] regulator: ti-abb: Fix timeout in ti_abb_wait_txdone/ti_abb_clear_all_txdone Greg Kroah-Hartman
2019-11-08 18:49 ` [PATCH 4.14 03/62] regulator: pfuze100-regulator: Variable "val" in pfuze100_regulator_probe() could be uninitialized Greg Kroah-Hartman
2019-11-08 18:49 ` [PATCH 4.14 04/62] ASoC: wm_adsp: Dont generate kcontrols without READ flags Greg Kroah-Hartman
2019-11-08 18:49 ` [PATCH 4.14 05/62] ASoc: rockchip: i2s: Fix RPM imbalance Greg Kroah-Hartman
2019-11-08 18:49 ` [PATCH 4.14 06/62] ARM: dts: logicpd-torpedo-som: Remove twl_keypad Greg Kroah-Hartman
2019-11-08 18:49 ` [PATCH 4.14 07/62] pinctrl: ns2: Fix off by one bugs in ns2_pinmux_enable() Greg Kroah-Hartman
2019-11-08 18:49 ` [PATCH 4.14 08/62] ARM: mm: fix alignment handler faults under memory pressure Greg Kroah-Hartman
2019-11-08 18:49 ` [PATCH 4.14 09/62] scsi: scsi_dh_alua: handle RTPG sense code correctly during state transitions Greg Kroah-Hartman
2019-11-08 18:49 ` [PATCH 4.14 10/62] scsi: sni_53c710: fix compilation error Greg Kroah-Hartman
2019-11-08 18:49 ` [PATCH 4.14 11/62] scsi: fix kconfig dependency warning related to 53C700_LE_ON_BE Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 12/62] ARM: dts: imx7s: Correct GPTs ipg clock source Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 13/62] perf c2c: Fix memory leak in build_cl_output() Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 14/62] perf kmem: Fix memory leak in compact_gfp_flags() Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 15/62] ARM: davinci: dm365: Fix McBSP dma_slave_map entry Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 16/62] scsi: target: core: Do not overwrite CDB byte 1 Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 17/62] ARM: 8926/1: v7m: remove register save to stack before svc Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 18/62] of: unittest: fix memory leak in unittest_data_add Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 19/62] MIPS: bmips: mark exception vectors as char arrays Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 20/62] i2c: stm32f7: remove warning when compiling with W=1 Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 21/62] cifs: Fix cifsInodeInfo lock_sem deadlock when reconnect occurs Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 22/62] nbd: handle racing with errored out commands Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 23/62] cxgb4: fix panic when attaching to ULD fail Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 24/62] dccp: do not leak jiffies on the wire Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 25/62] net: annotate accesses to sk->sk_incoming_cpu Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 26/62] net: annotate lockless accesses to sk->sk_napi_id Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 27/62] net: dsa: bcm_sf2: Fix IMP setup for port different than 8 Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 28/62] net: ethernet: ftgmac100: Fix DMA coherency issue with SW checksum Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 29/62] net: fix sk_page_frag() recursion from memory reclaim Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 30/62] net: hisilicon: Fix ping latency when deal with high throughput Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 31/62] net/mlx4_core: Dynamically set guaranteed amount of counters per VF Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 32/62] net: Zeroing the structure ethtool_wolinfo in ethtool_get_wol() Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 33/62] selftests: net: reuseport_dualstack: fix uninitalized parameter Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 34/62] udp: fix data-race in udp_set_dev_scratch() Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 35/62] net: add READ_ONCE() annotation in __skb_wait_for_more_packets() Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 36/62] net/mlx5e: Fix handling of compressed CQEs in case of low NAPI budget Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 37/62] net: dsa: b53: Do not clear existing mirrored port mask Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 38/62] net: usb: lan78xx: Connect PHY before registering MAC Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 39/62] r8152: add device id for Lenovo ThinkPad USB-C Dock Gen 2 Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 40/62] net: dsa: fix switch tree list Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 41/62] net: bcmgenet: reset 40nm EPHY on energy detect Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 42/62] net: add skb_queue_empty_lockless() Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 43/62] udp: use skb_queue_empty_lockless() Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 44/62] net: use skb_queue_empty_lockless() in poll() handlers Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 45/62] net: use skb_queue_empty_lockless() in busy poll contexts Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 46/62] vxlan: check tun_info options_len properly Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 47/62] erspan: fix the tun_info options_len check for erspan Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 48/62] inet: stop leaking jiffies on the wire Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 49/62] net/flow_dissector: switch to siphash Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 50/62] dmaengine: qcom: bam_dma: Fix resource leak Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 51/62] sched/wake_q: Fix wakeup ordering for wake_q Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 52/62] kbuild: use -fmacro-prefix-map to make __FILE__ a relative path Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 53/62] kbuild: add -fcf-protection=none when using retpoline flags Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 54/62] platform/x86: pmc_atom: Add Siemens SIMATIC IPC227E to critclk_systems DMI table Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 55/62] iio: adc: stm32-adc: move registers definitions Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 56/62] iio: adc: stm32-adc: fix a race when using several adcs with dma and irq Greg Kroah-Hartman
2019-11-08 18:50 ` Greg Kroah-Hartman [this message]
2019-11-08 18:50 ` [PATCH 4.14 58/62] powerpc/book3s64/mm: Dont do tlbie fixup for some hardware revisions Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 59/62] powerpc/book3s64/radix: Rename CPU_FTR_P9_TLBIE_BUG feature flag Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 60/62] powerpc/mm: Fixup tlbie vs mtpidr/mtlpidr ordering issue on POWER9 Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 61/62] selftests/powerpc: Add test case for tlbie vs mtpidr ordering issue Greg Kroah-Hartman
2019-11-08 18:50 ` [PATCH 4.14 62/62] selftests/powerpc: Fix compile error on tlbie_test due to newer gcc Greg Kroah-Hartman
2019-11-09  1:57 ` [PATCH 4.14 00/62] 4.14.153-stable review kernelci.org bot
2019-11-09 10:19 ` Naresh Kamboju
2019-11-09 15:40 ` Guenter Roeck

Reply instructions:

You may reply publically to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20191108174759.879781213@linuxfoundation.org \
    --to=gregkh@linuxfoundation.org \
    --cc=aneesh.kumar@linux.ibm.com \
    --cc=aneesh.kumar@linux.vnet.ibm.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linuxppc-dev@lists.ozlabs.org \
    --cc=mpe@ellerman.id.au \
    --cc=sandipan@linux.ibm.com \
    --cc=stable@vger.kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Stable Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/stable/0 stable/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 stable stable/ https://lore.kernel.org/stable \
		stable@vger.kernel.org
	public-inbox-index stable

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.stable


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git