From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S932798AbdCTSjn (ORCPT ); Mon, 20 Mar 2017 14:39:43 -0400 Received: from foss.arm.com ([217.140.101.70]:43014 "EHLO foss.arm.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1755890AbdCTRuj (ORCPT ); Mon, 20 Mar 2017 13:50:39 -0400 From: Marc Zyngier To: linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org Cc: Mark Rutland , Catalin Marinas , Daniel Lezcano , Will Deacon , Scott Wood , Hanjun Guo , Ding Tianhong , dann frazier Subject: [PATCH v2 13/18] arm64: arch_timer: Allows a CPU-specific erratum to only affect a subset of CPUs Date: Mon, 20 Mar 2017 17:48:24 +0000 Message-Id: <20170320174829.28182-14-marc.zyngier@arm.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20170320174829.28182-1-marc.zyngier@arm.com> References: <20170320174829.28182-1-marc.zyngier@arm.com> Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Instead of applying a CPU-specific workaround to all CPUs in the system, allow it to only affect a subset of them (typical big-little case). This is done by turning the erratum pointer into a per-CPU variable. Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/arch_timer.h | 31 ++++++++++++++++---------- drivers/clocksource/arm_arch_timer.c | 42 +++++++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index d46cf21e8b80..3cba3a5e4ba1 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -56,17 +57,25 @@ struct arch_timer_erratum_workaround { int (*set_next_event_virt)(unsigned long, struct clock_event_device *); }; -extern const struct arch_timer_erratum_workaround *timer_unstable_counter_workaround; - -#define arch_timer_reg_read_stable(reg) \ -({ \ - u64 _val; \ - if (needs_unstable_timer_counter_workaround() && \ - timer_unstable_counter_workaround->read_##reg) \ - _val = timer_unstable_counter_workaround->read_##reg(); \ - else \ - _val = read_sysreg(reg); \ - _val; \ +DECLARE_PER_CPU(const struct arch_timer_erratum_workaround *, + timer_unstable_counter_workaround); + +#define arch_timer_reg_read_stable(reg) \ +({ \ + u64 _val; \ + if (needs_unstable_timer_counter_workaround()) { \ + const struct arch_timer_erratum_workaround *wa; \ + preempt_disable(); \ + wa = __this_cpu_read(timer_unstable_counter_workaround); \ + if (wa && wa->read_##reg) \ + _val = wa->read_##reg(); \ + else \ + _val = read_sysreg(reg); \ + preempt_enable(); \ + } else { \ + _val = read_sysreg(reg); \ + } \ + _val; \ }) /* diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index b9f01daafdfa..2b17b99922a1 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -235,7 +235,8 @@ static u64 notrace hisi_161010101_read_cntvct_el0(void) #endif #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND -const struct arch_timer_erratum_workaround *timer_unstable_counter_workaround = NULL; +DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, + timer_unstable_counter_workaround); EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround); DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled); @@ -345,9 +346,18 @@ arch_timer_iterate_errata(enum arch_timer_erratum_match_type type, } static -void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa) +void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa, + bool local) { - timer_unstable_counter_workaround = wa; + int i; + + if (local) { + __this_cpu_write(timer_unstable_counter_workaround, wa); + } else { + for_each_possible_cpu(i) + per_cpu(timer_unstable_counter_workaround, i) = wa; + } + static_branch_enable(&arch_timer_read_ool_enabled); } @@ -376,14 +386,17 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t return; if (static_branch_unlikely(&arch_timer_read_ool_enabled)) { - if (wa != timer_unstable_counter_workaround) + const struct arch_timer_erratum_workaround *__wa; + __wa = __this_cpu_read(timer_unstable_counter_workaround); + if (__wa && wa != __wa) pr_warn("Can't enable workaround for %s (clashes with %s\n)", - wa->desc, - timer_unstable_counter_workaround->desc); - return; + wa->desc, __wa->desc); + + if (__wa) + return; } - arch_timer_enable_workaround(wa); + arch_timer_enable_workaround(wa, local); pr_info("Enabling %s workaround for %s\n", local ? "local" : "global", wa->desc); } @@ -391,10 +404,15 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t #define erratum_handler(fn, r, ...) \ ({ \ bool __val; \ - if (needs_unstable_timer_counter_workaround() && \ - timer_unstable_counter_workaround->fn) { \ - r = timer_unstable_counter_workaround->fn(__VA_ARGS__); \ - __val = true; \ + if (needs_unstable_timer_counter_workaround()) { \ + const struct arch_timer_erratum_workaround *__wa; \ + __wa = __this_cpu_read(timer_unstable_counter_workaround); \ + if (__wa && __wa->fn) { \ + r = __wa->fn(__VA_ARGS__); \ + __val = true; \ + } else { \ + __val = false; \ + } \ } else { \ __val = false; \ } \ -- 2.11.0 From mboxrd@z Thu Jan 1 00:00:00 1970 From: marc.zyngier@arm.com (Marc Zyngier) Date: Mon, 20 Mar 2017 17:48:24 +0000 Subject: [PATCH v2 13/18] arm64: arch_timer: Allows a CPU-specific erratum to only affect a subset of CPUs In-Reply-To: <20170320174829.28182-1-marc.zyngier@arm.com> References: <20170320174829.28182-1-marc.zyngier@arm.com> Message-ID: <20170320174829.28182-14-marc.zyngier@arm.com> To: linux-arm-kernel@lists.infradead.org List-Id: linux-arm-kernel.lists.infradead.org Instead of applying a CPU-specific workaround to all CPUs in the system, allow it to only affect a subset of them (typical big-little case). This is done by turning the erratum pointer into a per-CPU variable. Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/arch_timer.h | 31 ++++++++++++++++---------- drivers/clocksource/arm_arch_timer.c | 42 +++++++++++++++++++++++++----------- 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index d46cf21e8b80..3cba3a5e4ba1 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -56,17 +57,25 @@ struct arch_timer_erratum_workaround { int (*set_next_event_virt)(unsigned long, struct clock_event_device *); }; -extern const struct arch_timer_erratum_workaround *timer_unstable_counter_workaround; - -#define arch_timer_reg_read_stable(reg) \ -({ \ - u64 _val; \ - if (needs_unstable_timer_counter_workaround() && \ - timer_unstable_counter_workaround->read_##reg) \ - _val = timer_unstable_counter_workaround->read_##reg(); \ - else \ - _val = read_sysreg(reg); \ - _val; \ +DECLARE_PER_CPU(const struct arch_timer_erratum_workaround *, + timer_unstable_counter_workaround); + +#define arch_timer_reg_read_stable(reg) \ +({ \ + u64 _val; \ + if (needs_unstable_timer_counter_workaround()) { \ + const struct arch_timer_erratum_workaround *wa; \ + preempt_disable(); \ + wa = __this_cpu_read(timer_unstable_counter_workaround); \ + if (wa && wa->read_##reg) \ + _val = wa->read_##reg(); \ + else \ + _val = read_sysreg(reg); \ + preempt_enable(); \ + } else { \ + _val = read_sysreg(reg); \ + } \ + _val; \ }) /* diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index b9f01daafdfa..2b17b99922a1 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -235,7 +235,8 @@ static u64 notrace hisi_161010101_read_cntvct_el0(void) #endif #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND -const struct arch_timer_erratum_workaround *timer_unstable_counter_workaround = NULL; +DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, + timer_unstable_counter_workaround); EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround); DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled); @@ -345,9 +346,18 @@ arch_timer_iterate_errata(enum arch_timer_erratum_match_type type, } static -void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa) +void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa, + bool local) { - timer_unstable_counter_workaround = wa; + int i; + + if (local) { + __this_cpu_write(timer_unstable_counter_workaround, wa); + } else { + for_each_possible_cpu(i) + per_cpu(timer_unstable_counter_workaround, i) = wa; + } + static_branch_enable(&arch_timer_read_ool_enabled); } @@ -376,14 +386,17 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t return; if (static_branch_unlikely(&arch_timer_read_ool_enabled)) { - if (wa != timer_unstable_counter_workaround) + const struct arch_timer_erratum_workaround *__wa; + __wa = __this_cpu_read(timer_unstable_counter_workaround); + if (__wa && wa != __wa) pr_warn("Can't enable workaround for %s (clashes with %s\n)", - wa->desc, - timer_unstable_counter_workaround->desc); - return; + wa->desc, __wa->desc); + + if (__wa) + return; } - arch_timer_enable_workaround(wa); + arch_timer_enable_workaround(wa, local); pr_info("Enabling %s workaround for %s\n", local ? "local" : "global", wa->desc); } @@ -391,10 +404,15 @@ static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type t #define erratum_handler(fn, r, ...) \ ({ \ bool __val; \ - if (needs_unstable_timer_counter_workaround() && \ - timer_unstable_counter_workaround->fn) { \ - r = timer_unstable_counter_workaround->fn(__VA_ARGS__); \ - __val = true; \ + if (needs_unstable_timer_counter_workaround()) { \ + const struct arch_timer_erratum_workaround *__wa; \ + __wa = __this_cpu_read(timer_unstable_counter_workaround); \ + if (__wa && __wa->fn) { \ + r = __wa->fn(__VA_ARGS__); \ + __val = true; \ + } else { \ + __val = false; \ + } \ } else { \ __val = false; \ } \ -- 2.11.0