From: Peter Zijlstra <peterz@infradead.org>
To: dave.hansen@intel.com, luto@kernel.org, peterz@infradead.org
Cc: x86@kernel.org, Tom.StDenis@amd.com, linux-kernel@vger.kernel.org
Subject: [PATCH 3/4] x86/mm/cpa: Fold cpa_flush_range() and cpa_flush_array()
Date: Fri, 30 Nov 2018 14:44:58 +0100 [thread overview]
Message-ID: <20181130140337.605067461@infradead.org> (raw)
In-Reply-To: 20181130134455.490139778@infradead.org
[-- Attachment #1: peterz-cpa-fold-cpa_flush.patch --]
[-- Type: text/plain, Size: 4783 bytes --]
Note that the cache flush loop in cpa_flush_*() is identical when we
use __cpa_addr(); this then means the two functions are virtually
identical except for the TLB flushing.
Fold these two functions into a single cpa_flush() call.
One pesky detail is that __change_page_attr_set_clr() modifies @cpa,
so we have to save and restore part of that to ensure we flush the
original range.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
arch/x86/mm/pageattr.c | 96 ++++++++++++++-----------------------------------
1 file changed, 29 insertions(+), 67 deletions(-)
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -244,7 +244,7 @@ static unsigned long __cpa_addr(struct c
if (cpa->flags & CPA_ARRAY)
return cpa->vaddr[idx];
- return *cpa->vaddr;
+ return *cpa->vaddr + idx * PAGE_SIZE;
}
/*
@@ -304,50 +304,6 @@ static void cpa_flush_all(unsigned long
on_each_cpu(__cpa_flush_all, (void *) cache, 1);
}
-static bool cpa_check_flush_all(int cache)
-{
- BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);
-
- if (cache && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
- cpa_flush_all(cache);
- return true;
- }
-
- return false;
-}
-
-static void cpa_flush_range(unsigned long start, int numpages, int cache)
-{
- unsigned int i, level;
- unsigned long addr;
-
- WARN_ON(PAGE_ALIGN(start) != start);
-
- if (cpa_check_flush_all(cache))
- return;
-
- flush_tlb_kernel_range(start, start + PAGE_SIZE * numpages);
-
- if (!cache)
- return;
-
- /*
- * We only need to flush on one CPU,
- * clflush is a MESI-coherent instruction that
- * will cause all other CPUs to flush the same
- * cachelines:
- */
- for (i = 0, addr = start; i < numpages; i++, addr += PAGE_SIZE) {
- pte_t *pte = lookup_address(addr, &level);
-
- /*
- * Only flush present addresses:
- */
- if (pte && (pte_val(*pte) & _PAGE_PRESENT))
- clflush_cache_range((void *) addr, PAGE_SIZE);
- }
-}
-
void __cpa_flush_array(void *data)
{
struct cpa_data *cpa = data;
@@ -357,33 +313,37 @@ void __cpa_flush_array(void *data)
__flush_tlb_one_kernel(__cpa_addr(cpa, i));
}
-static void cpa_flush_array(struct cpa_data *cpa, int cache)
+static void cpa_flush(struct cpa_data *data, int cache)
{
+ struct cpa_data *cpa = data;
unsigned int i;
- if (cpa_check_flush_all(cache))
+ BUG_ON(irqs_disabled() && !early_boot_irqs_disabled);
+
+ if (cache && !static_cpu_has(X86_FEATURE_CLFLUSH)) {
+ cpa_flush_all(cache);
return;
+ }
- if (cpa->numpages <= tlb_single_page_flush_ceiling)
- on_each_cpu(__cpa_flush_array, cpa, 1);
- else
- flush_tlb_all();
+ if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
+ if (cpa->numpages <= tlb_single_page_flush_ceiling)
+ on_each_cpu(__cpa_flush_array, cpa, 1);
+ else
+ flush_tlb_all();
+ } else {
+ unsigned long start = __cpa_addr(cpa, 0);
+
+ flush_tlb_kernel_range(start, start + PAGE_SIZE * cpa->numpages);
+ }
if (!cache)
return;
- /*
- * We only need to flush on one CPU,
- * clflush is a MESI-coherent instruction that
- * will cause all other CPUs to flush the same
- * cachelines:
- */
for (i = 0; i < cpa->numpages; i++) {
unsigned long addr = __cpa_addr(cpa, i);
unsigned int level;
- pte_t *pte;
- pte = lookup_address(addr, &level);
+ pte_t *pte = lookup_address(addr, &level);
/*
* Only flush present addresses:
@@ -1695,7 +1655,7 @@ static int change_page_attr_set_clr(unsi
{
struct cpa_data cpa;
int ret, cache, checkalias;
- unsigned long baddr = 0;
+ unsigned long baddr = *addr;
memset(&cpa, 0, sizeof(cpa));
@@ -1781,11 +1741,11 @@ static int change_page_attr_set_clr(unsi
goto out;
}
- if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY))
- cpa_flush_array(&cpa, cache);
- else
- cpa_flush_range(baddr, numpages, cache);
+ /* Reset @cpa so that we flush the original range. */
+ cpa.vaddr = &baddr;
+ cpa.numpages = numpages;
+ cpa_flush(&cpa, cache);
out:
return ret;
}
@@ -2071,8 +2031,8 @@ int set_memory_global(unsigned long addr
static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc)
{
+ unsigned long start = addr;
struct cpa_data cpa;
- unsigned long start;
int ret;
/* Nothing to do if memory encryption is not active */
@@ -2099,7 +2059,7 @@ static int __set_memory_enc_dec(unsigned
/*
* Before changing the encryption attribute, we need to flush caches.
*/
- cpa_flush_range(start, numpages, 1);
+ cpa_flush(&cpa, 1);
ret = __change_page_attr_set_clr(&cpa, 1);
@@ -2110,7 +2070,9 @@ static int __set_memory_enc_dec(unsigned
* in case TLB flushing gets optimized in the cpa_flush_range()
* path use the same logic as above.
*/
- cpa_flush_range(start, numpages, 0);
+ cpa.vaddr = &start;
+ cpa.numpages = numpages;
+ cpa_flush(&cpa, 0);
return ret;
}
next prev parent reply other threads:[~2018-11-30 14:06 UTC|newest]
Thread overview: 22+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-11-30 13:44 [PATCH 0/4] x86/mm/cpa: Fix cpa-array TLB invalidation Peter Zijlstra
2018-11-30 13:44 ` [PATCH 1/4] x86/mm/cpa: Add __cpa_addr() helper Peter Zijlstra
2018-11-30 13:44 ` [PATCH 2/4] x86/mm/cpa: Fix cpa_flush_array() Peter Zijlstra
2018-11-30 17:43 ` Dave Hansen
2018-11-30 13:44 ` Peter Zijlstra [this message]
2018-11-30 13:44 ` [PATCH 4/4] x86/mm/cpa: Better use clflushopt Peter Zijlstra
2018-11-30 14:52 ` [PATCH 0/4] x86/mm/cpa: Fix cpa-array TLB invalidation StDenis, Tom
[not found] ` <BN6PR12MB180942F2C841FD138A046569F7D30@BN6PR12MB1809.namprd12.prod.outlook.com>
2018-11-30 15:08 ` StDenis, Tom
2018-11-30 15:09 ` Peter Zijlstra
2018-11-30 15:10 ` StDenis, Tom
2018-11-30 15:14 ` StDenis, Tom
2018-11-30 15:23 ` Peter Zijlstra
2018-11-30 15:27 ` StDenis, Tom
2018-11-30 17:50 ` Peter Zijlstra
2018-11-30 15:31 ` Peter Zijlstra
2018-11-30 16:19 ` StDenis, Tom
2018-11-30 17:48 ` Peter Zijlstra
2018-11-30 17:49 ` StDenis, Tom
2018-11-30 18:06 ` Peter Zijlstra
2018-12-03 15:41 ` Peter Zijlstra
2018-12-03 19:26 ` StDenis, Tom
2018-12-05 16:24 ` StDenis, Tom
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20181130140337.605067461@infradead.org \
--to=peterz@infradead.org \
--cc=Tom.StDenis@amd.com \
--cc=dave.hansen@intel.com \
--cc=linux-kernel@vger.kernel.org \
--cc=luto@kernel.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).