[v6,1/3] arm64: mte: make the per-task SCTLR_EL1 field usable elsewhere
Commit Message
In an upcoming change we are going to introduce per-task SCTLR_EL1
bits for PAC. Move the existing per-task SCTLR_EL1 field out of the
MTE-specific code so that we will be able to use it from both the
PAC and MTE code paths and make the task switching code more efficient.
Signed-off-by: Peter Collingbourne <pcc@google.com>
Link: https://linux-review.googlesource.com/id/Ic65fac78a7926168fa68f9e8da591c9e04ff7278
---
arch/arm64/Kconfig | 4 +++
arch/arm64/include/asm/mte.h | 4 ---
arch/arm64/include/asm/processor.h | 10 ++++++-
arch/arm64/kernel/mte.c | 47 ++++++------------------------
arch/arm64/kernel/process.c | 34 +++++++++++++++++----
5 files changed, 50 insertions(+), 49 deletions(-)
Comments
On Tue, Dec 29, 2020 at 10:59:13PM -0800, Peter Collingbourne wrote:
> In an upcoming change we are going to introduce per-task SCTLR_EL1
> bits for PAC. Move the existing per-task SCTLR_EL1 field out of the
> MTE-specific code so that we will be able to use it from both the
> PAC and MTE code paths and make the task switching code more efficient.
>
> Signed-off-by: Peter Collingbourne <pcc@google.com>
> Link: https://linux-review.googlesource.com/id/Ic65fac78a7926168fa68f9e8da591c9e04ff7278
> ---
> arch/arm64/Kconfig | 4 +++
> arch/arm64/include/asm/mte.h | 4 ---
> arch/arm64/include/asm/processor.h | 10 ++++++-
> arch/arm64/kernel/mte.c | 47 ++++++------------------------
> arch/arm64/kernel/process.c | 34 +++++++++++++++++----
> 5 files changed, 50 insertions(+), 49 deletions(-)
>
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index 05e17351e4f3..82e38d1ca012 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -348,6 +348,9 @@ config KASAN_SHADOW_OFFSET
> default 0xeffffff800000000 if ARM64_VA_BITS_36 && KASAN_SW_TAGS
> default 0xffffffffffffffff
>
> +config ARM64_NEED_SCTLR_USER
> + bool
> +
> source "arch/arm64/Kconfig.platforms"
>
> menu "Kernel Features"
> @@ -1653,6 +1656,7 @@ config ARM64_MTE
> # Required for tag checking in the uaccess routines
> depends on ARM64_PAN
> select ARCH_USES_HIGH_VMA_FLAGS
> + select ARM64_NEED_SCTLR_USER
I'm not sure I'd bother with this; is it the end of the world if we do this
unconditionally?
Will
On Tue, Jan 26, 2021 at 4:50 AM Will Deacon <will@kernel.org> wrote:
>
> On Tue, Dec 29, 2020 at 10:59:13PM -0800, Peter Collingbourne wrote:
> > In an upcoming change we are going to introduce per-task SCTLR_EL1
> > bits for PAC. Move the existing per-task SCTLR_EL1 field out of the
> > MTE-specific code so that we will be able to use it from both the
> > PAC and MTE code paths and make the task switching code more efficient.
> >
> > Signed-off-by: Peter Collingbourne <pcc@google.com>
> > Link: https://linux-review.googlesource.com/id/Ic65fac78a7926168fa68f9e8da591c9e04ff7278
> > ---
> > arch/arm64/Kconfig | 4 +++
> > arch/arm64/include/asm/mte.h | 4 ---
> > arch/arm64/include/asm/processor.h | 10 ++++++-
> > arch/arm64/kernel/mte.c | 47 ++++++------------------------
> > arch/arm64/kernel/process.c | 34 +++++++++++++++++----
> > 5 files changed, 50 insertions(+), 49 deletions(-)
> >
> > diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> > index 05e17351e4f3..82e38d1ca012 100644
> > --- a/arch/arm64/Kconfig
> > +++ b/arch/arm64/Kconfig
> > @@ -348,6 +348,9 @@ config KASAN_SHADOW_OFFSET
> > default 0xeffffff800000000 if ARM64_VA_BITS_36 && KASAN_SW_TAGS
> > default 0xffffffffffffffff
> >
> > +config ARM64_NEED_SCTLR_USER
> > + bool
> > +
> > source "arch/arm64/Kconfig.platforms"
> >
> > menu "Kernel Features"
> > @@ -1653,6 +1656,7 @@ config ARM64_MTE
> > # Required for tag checking in the uaccess routines
> > depends on ARM64_PAN
> > select ARCH_USES_HIGH_VMA_FLAGS
> > + select ARM64_NEED_SCTLR_USER
>
> I'm not sure I'd bother with this; is it the end of the world if we do this
> unconditionally?
I think I'd be fine with doing it unconditionally. If both PAC and MTE
are disabled then I believe that the only additional code that we
would end up executing is the comparison between prev and next
sctlr_user on task switch in process.c (which would always yield
false) and since task switch is much less common than entry/exit it
seems acceptable to me.
Peter
@@ -348,6 +348,9 @@ config KASAN_SHADOW_OFFSET
default 0xeffffff800000000 if ARM64_VA_BITS_36 && KASAN_SW_TAGS
default 0xffffffffffffffff
+config ARM64_NEED_SCTLR_USER
+ bool
+
source "arch/arm64/Kconfig.platforms"
menu "Kernel Features"
@@ -1653,6 +1656,7 @@ config ARM64_MTE
# Required for tag checking in the uaccess routines
depends on ARM64_PAN
select ARCH_USES_HIGH_VMA_FLAGS
+ select ARM64_NEED_SCTLR_USER
help
Memory Tagging (part of the ARMv8.5 Extensions) provides
architectural support for run-time, always-on detection of
@@ -42,7 +42,6 @@ void mte_free_tag_storage(char *storage);
void mte_sync_tags(pte_t *ptep, pte_t pte);
void mte_copy_page_tags(void *kto, const void *kfrom);
void flush_mte_state(void);
-void mte_thread_switch(struct task_struct *next);
void mte_suspend_exit(void);
long set_mte_ctrl(struct task_struct *task, unsigned long arg);
long get_mte_ctrl(struct task_struct *task);
@@ -65,9 +64,6 @@ static inline void mte_copy_page_tags(void *kto, const void *kfrom)
static inline void flush_mte_state(void)
{
}
-static inline void mte_thread_switch(struct task_struct *next)
-{
-}
static inline void mte_suspend_exit(void)
{
}
@@ -151,11 +151,15 @@ struct thread_struct {
struct ptrauth_keys_kernel keys_kernel;
#endif
#ifdef CONFIG_ARM64_MTE
- u64 sctlr_tcf0;
u64 gcr_user_excl;
#endif
+#ifdef CONFIG_ARM64_NEED_SCTLR_USER
+ u64 sctlr_user;
+#endif
};
+#define SCTLR_USER_MASK SCTLR_EL1_TCF0_MASK
+
static inline void arch_thread_struct_whitelist(unsigned long *offset,
unsigned long *size)
{
@@ -247,6 +251,10 @@ extern void release_thread(struct task_struct *);
unsigned long get_wchan(struct task_struct *p);
+#ifdef CONFIG_ARM64_NEED_SCTLR_USER
+void set_task_sctlr_el1(u64 sctlr);
+#endif
+
/* Thread switching */
extern struct task_struct *cpu_switch_to(struct task_struct *prev,
struct task_struct *next);
@@ -158,26 +158,6 @@ void mte_enable_kernel(void)
isb();
}
-static void update_sctlr_el1_tcf0(u64 tcf0)
-{
- /* ISB required for the kernel uaccess routines */
- sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF0_MASK, tcf0);
- isb();
-}
-
-static void set_sctlr_el1_tcf0(u64 tcf0)
-{
- /*
- * mte_thread_switch() checks current->thread.sctlr_tcf0 as an
- * optimisation. Disable preemption so that it does not see
- * the variable update before the SCTLR_EL1.TCF0 one.
- */
- preempt_disable();
- current->thread.sctlr_tcf0 = tcf0;
- update_sctlr_el1_tcf0(tcf0);
- preempt_enable();
-}
-
static void update_gcr_el1_excl(u64 excl)
{
@@ -210,21 +190,12 @@ void flush_mte_state(void)
write_sysreg_s(0, SYS_TFSRE0_EL1);
clear_thread_flag(TIF_MTE_ASYNC_FAULT);
/* disable tag checking */
- set_sctlr_el1_tcf0(SCTLR_EL1_TCF0_NONE);
+ set_task_sctlr_el1((current->thread.sctlr_user & ~SCTLR_EL1_TCF0_MASK) |
+ SCTLR_EL1_TCF0_NONE);
/* reset tag generation mask */
set_gcr_el1_excl(SYS_GCR_EL1_EXCL_MASK);
}
-void mte_thread_switch(struct task_struct *next)
-{
- if (!system_supports_mte())
- return;
-
- /* avoid expensive SCTLR_EL1 accesses if no change */
- if (current->thread.sctlr_tcf0 != next->thread.sctlr_tcf0)
- update_sctlr_el1_tcf0(next->thread.sctlr_tcf0);
-}
-
void mte_suspend_exit(void)
{
if (!system_supports_mte())
@@ -235,7 +206,7 @@ void mte_suspend_exit(void)
long set_mte_ctrl(struct task_struct *task, unsigned long arg)
{
- u64 tcf0;
+ u64 sctlr = task->thread.sctlr_user & ~SCTLR_EL1_TCF0_MASK;
u64 gcr_excl = ~((arg & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT) &
SYS_GCR_EL1_EXCL_MASK;
@@ -244,23 +215,23 @@ long set_mte_ctrl(struct task_struct *task, unsigned long arg)
switch (arg & PR_MTE_TCF_MASK) {
case PR_MTE_TCF_NONE:
- tcf0 = SCTLR_EL1_TCF0_NONE;
+ sctlr |= SCTLR_EL1_TCF0_NONE;
break;
case PR_MTE_TCF_SYNC:
- tcf0 = SCTLR_EL1_TCF0_SYNC;
+ sctlr |= SCTLR_EL1_TCF0_SYNC;
break;
case PR_MTE_TCF_ASYNC:
- tcf0 = SCTLR_EL1_TCF0_ASYNC;
+ sctlr |= SCTLR_EL1_TCF0_ASYNC;
break;
default:
return -EINVAL;
}
if (task != current) {
- task->thread.sctlr_tcf0 = tcf0;
+ task->thread.sctlr_user = sctlr;
task->thread.gcr_user_excl = gcr_excl;
} else {
- set_sctlr_el1_tcf0(tcf0);
+ set_task_sctlr_el1(sctlr);
set_gcr_el1_excl(gcr_excl);
}
@@ -277,7 +248,7 @@ long get_mte_ctrl(struct task_struct *task)
ret = incl << PR_MTE_TAG_SHIFT;
- switch (task->thread.sctlr_tcf0) {
+ switch (task->thread.sctlr_user & SCTLR_EL1_TCF0_MASK) {
case SCTLR_EL1_TCF0_NONE:
ret |= PR_MTE_TCF_NONE;
break;
@@ -529,6 +529,29 @@ static void erratum_1418040_thread_switch(struct task_struct *prev,
write_sysreg(val, cntkctl_el1);
}
+#ifdef CONFIG_ARM64_NEED_SCTLR_USER
+static void update_sctlr_el1(u64 sctlr)
+{
+ sysreg_clear_set(sctlr_el1, SCTLR_USER_MASK, sctlr);
+
+ /* ISB required for the kernel uaccess routines when setting TCF0. */
+ isb();
+}
+
+void set_task_sctlr_el1(u64 sctlr)
+{
+ /*
+ * __switch_to() checks current->thread.sctlr as an
+ * optimisation. Disable preemption so that it does not see
+ * the variable update before the SCTLR_EL1 one.
+ */
+ preempt_disable();
+ current->thread.sctlr_user = sctlr;
+ update_sctlr_el1(sctlr);
+ preempt_enable();
+}
+#endif /* CONFIG_ARM64_NEED_SCTLR_USER */
+
/*
* Thread switching.
*/
@@ -553,12 +576,11 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
*/
dsb(ish);
- /*
- * MTE thread switching must happen after the DSB above to ensure that
- * any asynchronous tag check faults have been logged in the TFSR*_EL1
- * registers.
- */
- mte_thread_switch(next);
+#ifdef CONFIG_ARM64_NEED_SCTLR_USER
+ /* avoid expensive SCTLR_EL1 accesses if no change */
+ if (prev->thread.sctlr_user != next->thread.sctlr_user)
+ update_sctlr_el1(next->thread.sctlr_user);
+#endif
/* the actual thread switch */
last = cpu_switch_to(prev, next);