From 97185c95f7ab7f752473c34672dab0925758094b Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 3 Apr 2015 12:02:02 +0200 Subject: x86/fpu: Split an fpstate_alloc_init() function out of init_fpu() Most init_fpu() users don't want the register-saving aspect of the function, they are calling it for 'current' and when FPU registers are not allocated and initialized yet. Split out a simplified API that does just that (and add debug-checks for these conditions): fpstate_alloc_init(). Use it where appropriate. Reviewed-by: Borislav Petkov Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/kernel/process.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 6e338e3b1dc0..abdb81d07423 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -159,7 +159,7 @@ void flush_thread(void) } else { if (!tsk_used_math(tsk)) { /* kthread execs. TODO: cleanup this horror. */ - if (WARN_ON(init_fpu(tsk))) + if (WARN_ON(fpstate_alloc_init(tsk))) force_sig(SIGKILL, tsk); user_fpu_begin(); } -- cgit v1.2.3 From ed97b085461ca6bdc22162b68b4cba69459ce1c8 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 3 Apr 2015 12:41:14 +0200 Subject: x86/fpu: Rename fpu_alloc() to fpstate_alloc() Use the fpu__*() namespace for fpstate_alloc() as well. Also add a comment about FPU state alignment. Reviewed-by: Borislav Petkov Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/include/asm/fpu-internal.h | 2 +- arch/x86/kernel/i387.c | 12 ++++++++---- arch/x86/kernel/process.c | 2 +- arch/x86/kvm/x86.c | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h index bdbba1a4de69..88f584ab3463 100644 --- a/arch/x86/include/asm/fpu-internal.h +++ b/arch/x86/include/asm/fpu-internal.h @@ -569,7 +569,7 @@ static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) } } -extern int fpu_alloc(struct fpu *fpu); +extern int fpstate_alloc(struct fpu *fpu); static inline void fpu_free(struct fpu *fpu) { diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c index 05fcc90087b0..5f2feb63b72a 100644 --- a/arch/x86/kernel/i387.c +++ b/arch/x86/kernel/i387.c @@ -246,17 +246,21 @@ void fpu_finit(struct fpu *fpu) } EXPORT_SYMBOL_GPL(fpu_finit); -int fpu_alloc(struct fpu *fpu) +int fpstate_alloc(struct fpu *fpu) { if (fpu->state) return 0; + fpu->state = kmem_cache_alloc(task_xstate_cachep, GFP_KERNEL); if (!fpu->state) return -ENOMEM; + + /* The CPU requires the FPU state to be aligned to 16 byte boundaries: */ WARN_ON((unsigned long)fpu->state & 15); + return 0; } -EXPORT_SYMBOL_GPL(fpu_alloc); +EXPORT_SYMBOL_GPL(fpstate_alloc); /* * Allocate the backing store for the current task's FPU registers @@ -276,7 +280,7 @@ int fpstate_alloc_init(struct task_struct *curr) /* * Memory allocation at the first usage of the FPU and other state. */ - ret = fpu_alloc(&curr->thread.fpu); + ret = fpstate_alloc(&curr->thread.fpu); if (ret) return ret; @@ -310,7 +314,7 @@ static int fpu__unlazy_stopped(struct task_struct *child) /* * Memory allocation at the first usage of the FPU and other state. */ - ret = fpu_alloc(&child->thread.fpu); + ret = fpstate_alloc(&child->thread.fpu); if (ret) return ret; diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index abdb81d07423..a0ed7c06a12c 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -92,7 +92,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) dst->thread.fpu.state = NULL; task_disable_lazy_fpu_restore(dst); if (tsk_used_math(src)) { - int err = fpu_alloc(&dst->thread.fpu); + int err = fpstate_alloc(&dst->thread.fpu); if (err) return err; fpu_copy(dst, src); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index bfc396632ee8..28fa733bb1c6 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7007,7 +7007,7 @@ int fx_init(struct kvm_vcpu *vcpu) { int err; - err = fpu_alloc(&vcpu->arch.guest_fpu); + err = fpstate_alloc(&vcpu->arch.guest_fpu); if (err) return err; -- cgit v1.2.3 From a7c2a83364c0d882a2d850c4c183c6de42d5c02b Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 3 Apr 2015 12:55:22 +0200 Subject: x86/fpu: Rename fpu_free() to fpstate_free() Use the fpu__*() namespace. Reviewed-by: Borislav Petkov Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/include/asm/fpu-internal.h | 2 +- arch/x86/kernel/process.c | 2 +- arch/x86/kvm/x86.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h index 88f584ab3463..b68e8f04c38a 100644 --- a/arch/x86/include/asm/fpu-internal.h +++ b/arch/x86/include/asm/fpu-internal.h @@ -571,7 +571,7 @@ static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) extern int fpstate_alloc(struct fpu *fpu); -static inline void fpu_free(struct fpu *fpu) +static inline void fpstate_free(struct fpu *fpu) { if (fpu->state) { kmem_cache_free(task_xstate_cachep, fpu->state); diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index a0ed7c06a12c..fd4aa56335de 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -102,7 +102,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) void free_thread_xstate(struct task_struct *tsk) { - fpu_free(&tsk->thread.fpu); + fpstate_free(&tsk->thread.fpu); } void arch_release_task_struct(struct task_struct *tsk) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 28fa733bb1c6..80a411c83083 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7029,7 +7029,7 @@ EXPORT_SYMBOL_GPL(fx_init); static void fx_free(struct kvm_vcpu *vcpu) { - fpu_free(&vcpu->arch.guest_fpu); + fpstate_free(&vcpu->arch.guest_fpu); } void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) -- cgit v1.2.3 From c0c2803dee21bef08ef5aacdf96fe2f1759ccc62 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 22 Apr 2015 09:52:56 +0200 Subject: x86/fpu: Move thread_info::fpu_counter into thread_info::fpu.counter This field is kept separate from the main FPU state structure for no good reason. Reviewed-by: Borislav Petkov Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/include/asm/fpu-internal.h | 10 +++++----- arch/x86/include/asm/processor.h | 18 +++++++++--------- arch/x86/kernel/process.c | 2 +- arch/x86/kernel/traps.c | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h index 02e0e97d8be7..f85d21b68901 100644 --- a/arch/x86/include/asm/fpu-internal.h +++ b/arch/x86/include/asm/fpu-internal.h @@ -384,7 +384,7 @@ static inline void drop_fpu(struct task_struct *tsk) * Forget coprocessor state.. */ preempt_disable(); - tsk->thread.fpu_counter = 0; + tsk->thread.fpu.counter = 0; if (__thread_has_fpu(tsk)) { /* Ignore delayed exceptions from user space */ @@ -441,7 +441,7 @@ static inline fpu_switch_t switch_fpu_prepare(struct task_struct *old, struct ta * or if the past 5 consecutive context-switches used math. */ fpu.preload = tsk_used_math(new) && - (use_eager_fpu() || new->thread.fpu_counter > 5); + (use_eager_fpu() || new->thread.fpu.counter > 5); if (__thread_has_fpu(old)) { if (!__save_init_fpu(old)) @@ -454,16 +454,16 @@ static inline fpu_switch_t switch_fpu_prepare(struct task_struct *old, struct ta /* Don't change CR0.TS if we just switch! */ if (fpu.preload) { - new->thread.fpu_counter++; + new->thread.fpu.counter++; __thread_set_has_fpu(new); prefetch(new->thread.fpu.state); } else if (!use_eager_fpu()) stts(); } else { - old->thread.fpu_counter = 0; + old->thread.fpu.counter = 0; task_disable_lazy_fpu_restore(old); if (fpu.preload) { - new->thread.fpu_counter++; + new->thread.fpu.counter++; if (fpu_lazy_restore(new, cpu)) fpu.preload = 0; else diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 2dc08c231a9a..64d6b5d97ce9 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -433,6 +433,15 @@ struct fpu { unsigned int last_cpu; unsigned int has_fpu; union thread_xstate *state; + /* + * This counter contains the number of consecutive context switches + * that the FPU is used. If this is over a threshold, the lazy fpu + * saving becomes unlazy to save the trap. This is an unsigned char + * so that after 256 times the counter wraps and the behavior turns + * lazy again; this to deal with bursty apps that only use FPU for + * a short time + */ + unsigned char counter; }; #ifdef CONFIG_X86_64 @@ -535,15 +544,6 @@ struct thread_struct { unsigned long iopl; /* Max allowed port in the bitmap, in bytes: */ unsigned io_bitmap_max; - /* - * fpu_counter contains the number of consecutive context switches - * that the FPU is used. If this is over a threshold, the lazy fpu - * saving becomes unlazy to save the trap. This is an unsigned char - * so that after 256 times the counter wraps and the behavior turns - * lazy again; this to deal with bursty apps that only use FPU for - * a short time - */ - unsigned char fpu_counter; }; /* diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index fd4aa56335de..c7793addc237 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -87,7 +87,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { *dst = *src; - dst->thread.fpu_counter = 0; + dst->thread.fpu.counter = 0; dst->thread.fpu.has_fpu = 0; dst->thread.fpu.state = NULL; task_disable_lazy_fpu_restore(dst); diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index cf9c9627be19..231aa579d9cd 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -863,7 +863,7 @@ void math_state_restore(void) fpu_reset_state(tsk); force_sig_info(SIGSEGV, SEND_SIG_PRIV, tsk); } else { - tsk->thread.fpu_counter++; + tsk->thread.fpu.counter++; } kernel_fpu_enable(); } -- cgit v1.2.3 From f89e32e0a3df2f29d61fdc120ac62654ef267111 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 22 Apr 2015 10:58:10 +0200 Subject: x86/fpu: Fix header file dependencies of fpu-internal.h Fix a minor header file dependency bug in asm/fpu-internal.h: it relies on i387.h but does not include it. All users of fpu-internal.h included it explicitly. Also remove unnecessary includes, to reduce compilation time. This also makes it easier to use it as a standalone header file for FPU internals, such as an upcoming C module in arch/x86/kernel/fpu/. Reviewed-by: Borislav Petkov Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/crypto/crc32c-intel_glue.c | 1 - arch/x86/crypto/sha-mb/sha1_mb.c | 1 - arch/x86/ia32/ia32_signal.c | 1 - arch/x86/include/asm/fpu-internal.h | 9 ++------- arch/x86/kernel/cpu/common.c | 1 - arch/x86/kernel/process.c | 1 - arch/x86/kernel/process_32.c | 1 - arch/x86/kernel/process_64.c | 1 - arch/x86/kernel/ptrace.c | 1 - arch/x86/kernel/signal.c | 1 - arch/x86/kernel/smpboot.c | 1 - arch/x86/kernel/traps.c | 1 - arch/x86/kvm/x86.c | 2 +- arch/x86/mm/mpx.c | 1 - 14 files changed, 3 insertions(+), 20 deletions(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/crypto/crc32c-intel_glue.c b/arch/x86/crypto/crc32c-intel_glue.c index 28640c3d6af7..470522cb042a 100644 --- a/arch/x86/crypto/crc32c-intel_glue.c +++ b/arch/x86/crypto/crc32c-intel_glue.c @@ -32,7 +32,6 @@ #include #include -#include #include #define CHKSUM_BLOCK_SIZE 1 diff --git a/arch/x86/crypto/sha-mb/sha1_mb.c b/arch/x86/crypto/sha-mb/sha1_mb.c index e510b1c5d690..15373786494f 100644 --- a/arch/x86/crypto/sha-mb/sha1_mb.c +++ b/arch/x86/crypto/sha-mb/sha1_mb.c @@ -65,7 +65,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/x86/ia32/ia32_signal.c b/arch/x86/ia32/ia32_signal.c index c81d35e6c7f1..4bafd5b05aca 100644 --- a/arch/x86/ia32/ia32_signal.c +++ b/arch/x86/ia32/ia32_signal.c @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h index f85d21b68901..c3b7bd12f18f 100644 --- a/arch/x86/include/asm/fpu-internal.h +++ b/arch/x86/include/asm/fpu-internal.h @@ -10,18 +10,13 @@ #ifndef _FPU_INTERNAL_H #define _FPU_INTERNAL_H -#include #include #include #include -#include -#include -#include -#include + #include -#include +#include #include -#include #ifdef CONFIG_X86_64 # include diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index b8035b8dd186..220ad95e0e28 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index c7793addc237..35d0f1925524 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c index 8ed2106b06da..84d647d4b14d 100644 --- a/arch/x86/kernel/process_32.c +++ b/arch/x86/kernel/process_32.c @@ -39,7 +39,6 @@ #include #include #include -#include #include #include #ifdef CONFIG_MATH_EMULATION diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index ddfdbf74f174..ae6efeccb46e 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -38,7 +38,6 @@ #include #include -#include #include #include #include diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c index a7bc79480719..69451b8965f7 100644 --- a/arch/x86/kernel/ptrace.c +++ b/arch/x86/kernel/ptrace.c @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c index 1ea14fd53933..35f867aa597e 100644 --- a/arch/x86/kernel/signal.c +++ b/arch/x86/kernel/signal.c @@ -26,7 +26,6 @@ #include #include -#include #include #include #include diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 50e547eac8cd..60e331ceb844 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -68,7 +68,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 231aa579d9cd..465b335e7491 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -54,7 +54,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 26b1f89fc608..be276e0fe0ff 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -59,7 +59,7 @@ #include #include #include -#include +#include #include /* Ugh! */ #include #include diff --git a/arch/x86/mm/mpx.c b/arch/x86/mm/mpx.c index c439ec478216..37ad432e7f16 100644 --- a/arch/x86/mm/mpx.c +++ b/arch/x86/mm/mpx.c @@ -10,7 +10,6 @@ #include #include -#include #include #include #include -- cgit v1.2.3 From 11ad19277e025f914518bc2943a240cdd37cf844 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 22 Apr 2015 11:44:46 +0200 Subject: x86/fpu: Remove the free_thread_xstate() complication Use fpstate_free() directly to manage FPU state. Only process.c was using this method, so this is a speedup as well, as it removes the extra function call and related clobbers. Reviewed-by: Borislav Petkov Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/include/asm/processor.h | 1 - arch/x86/kernel/process.c | 9 ++------- 2 files changed, 2 insertions(+), 8 deletions(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 6b75c4b927ec..fef8db024ece 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -362,7 +362,6 @@ DECLARE_PER_CPU(struct irq_stack *, softirq_stack); #endif /* X86_64 */ extern unsigned int xstate_size; -extern void free_thread_xstate(struct task_struct *); extern struct kmem_cache *task_xstate_cachep; struct perf_event; diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 35d0f1925524..6ab180f40a7e 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -99,14 +99,9 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) return 0; } -void free_thread_xstate(struct task_struct *tsk) -{ - fpstate_free(&tsk->thread.fpu); -} - void arch_release_task_struct(struct task_struct *tsk) { - free_thread_xstate(tsk); + fpstate_free(&tsk->thread.fpu); } void arch_task_cache_init(void) @@ -154,7 +149,7 @@ void flush_thread(void) if (!use_eager_fpu()) { /* FPU state will be reallocated lazily at the first use. */ drop_fpu(tsk); - free_thread_xstate(tsk); + fpstate_free(&tsk->thread.fpu); } else { if (!tsk_used_math(tsk)) { /* kthread execs. TODO: cleanup this horror. */ -- cgit v1.2.3 From 81683cc8277e79decff4d0cf82ae0e17d2fe465f Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 22 Apr 2015 11:52:13 +0200 Subject: x86/fpu: Factor out fpu__flush_thread() from flush_thread() flush_thread() open codes a lot of FPU internals - create a separate function for it in fpu/core.c. Turns out that this does not hurt performance: text data bss dec hex filename 11843039 1884440 1130496 14857975 e2b6f7 vmlinux.before 11843039 1884440 1130496 14857975 e2b6f7 vmlinux.after and since this is a slowpath clarity comes first anyway. We can reconsider inlining decisions after the FPU code has been cleaned up. Reviewed-by: Borislav Petkov Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/include/asm/i387.h | 1 + arch/x86/kernel/fpu/core.c | 17 +++++++++++++++++ arch/x86/kernel/process.c | 14 +------------- 3 files changed, 19 insertions(+), 13 deletions(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/include/asm/i387.h b/arch/x86/include/asm/i387.h index 6552a16e0e38..d6fc84440b73 100644 --- a/arch/x86/include/asm/i387.h +++ b/arch/x86/include/asm/i387.h @@ -20,6 +20,7 @@ struct user_i387_struct; extern int fpstate_alloc_init(struct task_struct *curr); extern void fpstate_init(struct fpu *fpu); +extern void fpu__flush_thread(struct task_struct *tsk); extern int dump_fpu(struct pt_regs *, struct user_i387_struct *); extern void math_state_restore(void); diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 9211582f5d3f..787bf57b8422 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -227,6 +227,23 @@ static int fpu__unlazy_stopped(struct task_struct *child) return 0; } +void fpu__flush_thread(struct task_struct *tsk) +{ + if (!use_eager_fpu()) { + /* FPU state will be reallocated lazily at the first use. */ + drop_fpu(tsk); + fpstate_free(&tsk->thread.fpu); + } else { + if (!tsk_used_math(tsk)) { + /* kthread execs. TODO: cleanup this horror. */ + if (WARN_ON(fpstate_alloc_init(tsk))) + force_sig(SIGKILL, tsk); + user_fpu_begin(); + } + restore_init_xstate(); + } +} + /* * The xstateregs_active() routine is the same as the fpregs_active() routine, * as the "regset->n" for the xstate regset will be updated based on the feature diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 6ab180f40a7e..52fd8f6f44c7 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -146,19 +146,7 @@ void flush_thread(void) flush_ptrace_hw_breakpoint(tsk); memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array)); - if (!use_eager_fpu()) { - /* FPU state will be reallocated lazily at the first use. */ - drop_fpu(tsk); - fpstate_free(&tsk->thread.fpu); - } else { - if (!tsk_used_math(tsk)) { - /* kthread execs. TODO: cleanup this horror. */ - if (WARN_ON(fpstate_alloc_init(tsk))) - force_sig(SIGKILL, tsk); - user_fpu_begin(); - } - restore_init_xstate(); - } + fpu__flush_thread(tsk); } static void hard_disable_TSC(void) -- cgit v1.2.3 From 8ffb53ab986ccb4421b1060182c6e084edd7b9d8 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 22 Apr 2015 15:41:56 +0200 Subject: x86/fpu: Move task_xstate_cachep handling to core.c This code was historically in process.c, now we have FPU core internals in fpu/core.c instead - move it there. Reviewed-by: Borislav Petkov Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/include/asm/fpu-internal.h | 2 ++ arch/x86/kernel/fpu/core.c | 15 +++++++++++++++ arch/x86/kernel/process.c | 9 +-------- 3 files changed, 18 insertions(+), 8 deletions(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h index 95e04cb1ed2f..f41170c6d376 100644 --- a/arch/x86/include/asm/fpu-internal.h +++ b/arch/x86/include/asm/fpu-internal.h @@ -564,6 +564,8 @@ static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) } } +extern void fpstate_cache_init(void); + extern int fpstate_alloc(struct fpu *fpu); static inline void fpstate_free(struct fpu *fpu) diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 15c3cf7bd160..b32a6eb7f189 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -147,6 +147,21 @@ void fpstate_init(struct fpu *fpu) } EXPORT_SYMBOL_GPL(fpstate_init); +/* + * FPU state allocation: + */ +struct kmem_cache *task_xstate_cachep; +EXPORT_SYMBOL_GPL(task_xstate_cachep); + +void fpstate_cache_init(void) +{ + task_xstate_cachep = + kmem_cache_create("task_xstate", xstate_size, + __alignof__(union thread_xstate), + SLAB_PANIC | SLAB_NOTRACK, NULL); + setup_xstate_comp(); +} + int fpstate_alloc(struct fpu *fpu) { if (fpu->state) diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 52fd8f6f44c7..36d9f737f278 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -75,9 +75,6 @@ void idle_notifier_unregister(struct notifier_block *n) EXPORT_SYMBOL_GPL(idle_notifier_unregister); #endif -struct kmem_cache *task_xstate_cachep; -EXPORT_SYMBOL_GPL(task_xstate_cachep); - /* * this gets called so that we can store lazy state into memory and copy the * current task into the new thread. @@ -106,11 +103,7 @@ void arch_release_task_struct(struct task_struct *tsk) void arch_task_cache_init(void) { - task_xstate_cachep = - kmem_cache_create("task_xstate", xstate_size, - __alignof__(union thread_xstate), - SLAB_PANIC | SLAB_NOTRACK, NULL); - setup_xstate_comp(); + fpstate_cache_init(); } /* -- cgit v1.2.3 From a752b53d9dcbae28a3a22b5577f0571acf53d5aa Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 22 Apr 2015 15:47:05 +0200 Subject: x86/fpu: Factor out fpu__copy() Introduce fpu__copy() and use it in arch_dup_task_struct(), thus moving another chunk of FPU logic to fpu/core.c. Reviewed-by: Borislav Petkov Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/include/asm/fpu-internal.h | 1 + arch/x86/kernel/fpu/core.c | 18 ++++++++++++++++++ arch/x86/kernel/process.c | 12 +----------- 3 files changed, 20 insertions(+), 11 deletions(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h index f41170c6d376..6c1ceb7c3f9a 100644 --- a/arch/x86/include/asm/fpu-internal.h +++ b/arch/x86/include/asm/fpu-internal.h @@ -567,6 +567,7 @@ static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) extern void fpstate_cache_init(void); extern int fpstate_alloc(struct fpu *fpu); +extern int fpu__copy(struct task_struct *dst, struct task_struct *src); static inline void fpstate_free(struct fpu *fpu) { diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index b32a6eb7f189..05df212449ed 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -178,6 +178,24 @@ int fpstate_alloc(struct fpu *fpu) } EXPORT_SYMBOL_GPL(fpstate_alloc); +int fpu__copy(struct task_struct *dst, struct task_struct *src) +{ + dst->thread.fpu.counter = 0; + dst->thread.fpu.has_fpu = 0; + dst->thread.fpu.state = NULL; + + task_disable_lazy_fpu_restore(dst); + + if (tsk_used_math(src)) { + int err = fpstate_alloc(&dst->thread.fpu); + + if (err) + return err; + fpu_copy(dst, src); + } + return 0; +} + /* * Allocate the backing store for the current task's FPU registers * and initialize the registers themselves as well. diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 36d9f737f278..bb7d4abcdad6 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -83,17 +83,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { *dst = *src; - dst->thread.fpu.counter = 0; - dst->thread.fpu.has_fpu = 0; - dst->thread.fpu.state = NULL; - task_disable_lazy_fpu_restore(dst); - if (tsk_used_math(src)) { - int err = fpstate_alloc(&dst->thread.fpu); - if (err) - return err; - fpu_copy(dst, src); - } - return 0; + return fpu__copy(dst, src); } void arch_release_task_struct(struct task_struct *tsk) -- cgit v1.2.3 From ca6787ba0fcc875cfb06dc2a538ac23210b7d251 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Thu, 23 Apr 2015 12:33:50 +0200 Subject: x86/fpu: Remove 'struct task_struct' usage from drop_fpu() Migrate this function to pure 'struct fpu' usage. Reviewed-by: Borislav Petkov Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/include/asm/fpu-internal.h | 9 +++++---- arch/x86/kernel/fpu/core.c | 2 +- arch/x86/kernel/fpu/xsave.c | 2 +- arch/x86/kernel/process.c | 3 ++- 4 files changed, 9 insertions(+), 7 deletions(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h index 9311126571ab..e8f7134f0ffb 100644 --- a/arch/x86/include/asm/fpu-internal.h +++ b/arch/x86/include/asm/fpu-internal.h @@ -358,14 +358,13 @@ static inline void __thread_fpu_begin(struct fpu *fpu) __thread_set_has_fpu(fpu); } -static inline void drop_fpu(struct task_struct *tsk) +static inline void drop_fpu(struct fpu *fpu) { - struct fpu *fpu = &tsk->thread.fpu; /* * Forget coprocessor state.. */ preempt_disable(); - tsk->thread.fpu.counter = 0; + fpu->counter = 0; if (fpu->has_fpu) { /* Ignore delayed exceptions from user space */ @@ -394,8 +393,10 @@ static inline void restore_init_xstate(void) */ static inline void fpu_reset_state(struct task_struct *tsk) { + struct fpu *fpu = &tsk->thread.fpu; + if (!use_eager_fpu()) - drop_fpu(tsk); + drop_fpu(fpu); else restore_init_xstate(); } diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 9e7f9e7b2cca..ba539fc018d7 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -389,7 +389,7 @@ void fpu__flush_thread(struct task_struct *tsk) if (!use_eager_fpu()) { /* FPU state will be reallocated lazily at the first use. */ - drop_fpu(tsk); + drop_fpu(fpu); fpstate_free(&tsk->thread.fpu); } else { if (!fpu->fpstate_active) { diff --git a/arch/x86/kernel/fpu/xsave.c b/arch/x86/kernel/fpu/xsave.c index dc346e19c0df..049dc619481d 100644 --- a/arch/x86/kernel/fpu/xsave.c +++ b/arch/x86/kernel/fpu/xsave.c @@ -392,7 +392,7 @@ int __restore_xstate_sig(void __user *buf, void __user *buf_fx, int size) * We will be ready to restore/save the state only after * fpu->fpstate_active is again set. */ - drop_fpu(tsk); + drop_fpu(fpu); if (__copy_from_user(&fpu->state->xsave, buf_fx, state_size) || __copy_from_user(&env, buf, sizeof(env))) { diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index bb7d4abcdad6..50d503a2d8c3 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -104,6 +104,7 @@ void exit_thread(void) struct task_struct *me = current; struct thread_struct *t = &me->thread; unsigned long *bp = t->io_bitmap_ptr; + struct fpu *fpu = &t->fpu; if (bp) { struct tss_struct *tss = &per_cpu(cpu_tss, get_cpu()); @@ -119,7 +120,7 @@ void exit_thread(void) kfree(bp); } - drop_fpu(me); + drop_fpu(fpu); } void flush_thread(void) -- cgit v1.2.3 From c69e098b1f90c0d520c4d5b5bff9f2ede95b13a8 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 24 Apr 2015 02:07:15 +0200 Subject: x86/fpu: Use 'struct fpu' in fpu__copy() Migrate this function to pure 'struct fpu' usage. Reviewed-by: Borislav Petkov Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/include/asm/fpu-internal.h | 2 +- arch/x86/kernel/fpu/core.c | 13 +++++-------- arch/x86/kernel/process.c | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h index 6b84399c8839..21ad68179454 100644 --- a/arch/x86/include/asm/fpu-internal.h +++ b/arch/x86/include/asm/fpu-internal.h @@ -535,7 +535,7 @@ extern void fpstate_cache_init(void); extern int fpstate_alloc(struct fpu *fpu); extern void fpstate_free(struct fpu *fpu); -extern int fpu__copy(struct task_struct *dst, struct task_struct *src); +extern int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu); static inline unsigned long alloc_mathframe(unsigned long sp, int ia32_frame, unsigned long *buf_fx, diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 9aaba6abfae3..a84358575235 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -240,15 +240,12 @@ static void fpu_copy(struct fpu *dst_fpu, struct fpu *src_fpu) } } -int fpu__copy(struct task_struct *dst, struct task_struct *src) +int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu) { - struct fpu *dst_fpu = &dst->thread.fpu; - struct fpu *src_fpu = &src->thread.fpu; - - dst->thread.fpu.counter = 0; - dst->thread.fpu.has_fpu = 0; - dst->thread.fpu.state = NULL; - dst->thread.fpu.last_cpu = -1; + dst_fpu->counter = 0; + dst_fpu->has_fpu = 0; + dst_fpu->state = NULL; + dst_fpu->last_cpu = -1; if (src_fpu->fpstate_active) { int err = fpstate_alloc(dst_fpu); diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 50d503a2d8c3..e97266b18ad3 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -83,7 +83,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { *dst = *src; - return fpu__copy(dst, src); + return fpu__copy(&dst->thread.fpu, &src->thread.fpu); } void arch_release_task_struct(struct task_struct *tsk) -- cgit v1.2.3 From 2e8a3102662233dfac92fe70f56429b4050f674a Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 24 Apr 2015 02:28:23 +0200 Subject: x86/fpu: Rename fpu__flush_thread() to fpu__clear() The primary purpose of this function is to clear the current task's FPU before an exec(), to not leak information from the previous task, and to allow the new task to start with freshly initialized FPU registers. Rename the function to reflect this primary purpose. Reviewed-by: Borislav Petkov Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/include/asm/i387.h | 2 +- arch/x86/kernel/fpu/core.c | 4 ++-- arch/x86/kernel/process.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/include/asm/i387.h b/arch/x86/include/asm/i387.h index 38376cdf297c..b8f7d76ac066 100644 --- a/arch/x86/include/asm/i387.h +++ b/arch/x86/include/asm/i387.h @@ -20,7 +20,7 @@ struct user_i387_struct; extern int fpstate_alloc_init(struct fpu *fpu); extern void fpstate_init(struct fpu *fpu); -extern void fpu__flush_thread(struct task_struct *tsk); +extern void fpu__clear(struct task_struct *tsk); extern int dump_fpu(struct pt_regs *, struct user_i387_struct *); extern void fpu__restore(void); diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index e3e8585284ad..c15d064ce43e 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -381,11 +381,11 @@ void fpu__restore(void) } EXPORT_SYMBOL_GPL(fpu__restore); -void fpu__flush_thread(struct task_struct *tsk) +void fpu__clear(struct task_struct *tsk) { struct fpu *fpu = &tsk->thread.fpu; - WARN_ON(tsk != current); + WARN_ON_ONCE(tsk != current); /* Almost certainly an anomaly */ if (!use_eager_fpu()) { /* FPU state will be reallocated lazily at the first use. */ diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index e97266b18ad3..51ad3422e728 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -130,7 +130,7 @@ void flush_thread(void) flush_ptrace_hw_breakpoint(tsk); memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array)); - fpu__flush_thread(tsk); + fpu__clear(tsk); } static void hard_disable_TSC(void) -- cgit v1.2.3 From 78f7f1e54bac032b98956862a5bcf8c28ed22d07 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 24 Apr 2015 02:54:44 +0200 Subject: x86/fpu: Rename fpu-internal.h to fpu/internal.h This unifies all the FPU related header files under a unified, hiearchical naming scheme: - asm/fpu/types.h: FPU related data types, needed for 'struct task_struct', widely included in almost all kernel code, and hence kept as small as possible. - asm/fpu/api.h: FPU related 'public' methods exported to other subsystems. - asm/fpu/internal.h: FPU subsystem internal methods - asm/fpu/xsave.h: XSAVE support internal methods (Also standardize the header guard in asm/fpu/internal.h.) Reviewed-by: Borislav Petkov Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/crypto/crc32c-intel_glue.c | 2 +- arch/x86/crypto/sha-mb/sha1_mb.c | 2 +- arch/x86/ia32/ia32_signal.c | 2 +- arch/x86/include/asm/fpu-internal.h | 556 ------------------------------------ arch/x86/include/asm/fpu/internal.h | 556 ++++++++++++++++++++++++++++++++++++ arch/x86/kernel/cpu/common.c | 2 +- arch/x86/kernel/fpu/core.c | 2 +- arch/x86/kernel/fpu/init.c | 2 +- arch/x86/kernel/fpu/xsave.c | 2 +- arch/x86/kernel/process.c | 2 +- arch/x86/kernel/process_32.c | 2 +- arch/x86/kernel/process_64.c | 2 +- arch/x86/kernel/ptrace.c | 2 +- arch/x86/kernel/signal.c | 2 +- arch/x86/kernel/smpboot.c | 2 +- arch/x86/kernel/traps.c | 2 +- arch/x86/kvm/x86.c | 2 +- arch/x86/mm/mpx.c | 2 +- arch/x86/power/cpu.c | 2 +- 19 files changed, 573 insertions(+), 573 deletions(-) delete mode 100644 arch/x86/include/asm/fpu-internal.h create mode 100644 arch/x86/include/asm/fpu/internal.h (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/crypto/crc32c-intel_glue.c b/arch/x86/crypto/crc32c-intel_glue.c index 470522cb042a..81a595d75cf5 100644 --- a/arch/x86/crypto/crc32c-intel_glue.c +++ b/arch/x86/crypto/crc32c-intel_glue.c @@ -32,7 +32,7 @@ #include #include -#include +#include #define CHKSUM_BLOCK_SIZE 1 #define CHKSUM_DIGEST_SIZE 4 diff --git a/arch/x86/crypto/sha-mb/sha1_mb.c b/arch/x86/crypto/sha-mb/sha1_mb.c index 02b64bbc1d48..03ffaf8c2244 100644 --- a/arch/x86/crypto/sha-mb/sha1_mb.c +++ b/arch/x86/crypto/sha-mb/sha1_mb.c @@ -68,7 +68,7 @@ #include #include #include -#include +#include #include "sha_mb_ctx.h" #define FLUSH_INTERVAL 1000 /* in usec */ diff --git a/arch/x86/ia32/ia32_signal.c b/arch/x86/ia32/ia32_signal.c index e1ec6f90d09e..d6d8f4ca5136 100644 --- a/arch/x86/ia32/ia32_signal.c +++ b/arch/x86/ia32/ia32_signal.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h deleted file mode 100644 index 20690a14c73a..000000000000 --- a/arch/x86/include/asm/fpu-internal.h +++ /dev/null @@ -1,556 +0,0 @@ -/* - * Copyright (C) 1994 Linus Torvalds - * - * Pentium III FXSR, SSE support - * General FPU state handling cleanups - * Gareth Hughes , May 2000 - * x86-64 work by Andi Kleen 2002 - */ - -#ifndef _FPU_INTERNAL_H -#define _FPU_INTERNAL_H - -#include -#include -#include - -#include -#include -#include - -#ifdef CONFIG_X86_64 -# include -# include -struct ksignal; -int ia32_setup_rt_frame(int sig, struct ksignal *ksig, - compat_sigset_t *set, struct pt_regs *regs); -int ia32_setup_frame(int sig, struct ksignal *ksig, - compat_sigset_t *set, struct pt_regs *regs); -#else -# define user_i387_ia32_struct user_i387_struct -# define user32_fxsr_struct user_fxsr_struct -# define ia32_setup_frame __setup_frame -# define ia32_setup_rt_frame __setup_rt_frame -#endif - -extern unsigned int mxcsr_feature_mask; -extern void fpu__cpu_init(void); -extern void eager_fpu_init(void); - -DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); - -extern void convert_from_fxsr(struct user_i387_ia32_struct *env, - struct task_struct *tsk); -extern void convert_to_fxsr(struct task_struct *tsk, - const struct user_i387_ia32_struct *env); - -extern user_regset_active_fn fpregs_active, xfpregs_active; -extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get, - xstateregs_get; -extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set, - xstateregs_set; - -/* - * xstateregs_active == fpregs_active. Please refer to the comment - * at the definition of fpregs_active. - */ -#define xstateregs_active fpregs_active - -#ifdef CONFIG_MATH_EMULATION -extern void finit_soft_fpu(struct i387_soft_struct *soft); -#else -static inline void finit_soft_fpu(struct i387_soft_struct *soft) {} -#endif - -/* - * Must be run with preemption disabled: this clears the fpu_fpregs_owner_ctx, - * on this CPU. - * - * This will disable any lazy FPU state restore of the current FPU state, - * but if the current thread owns the FPU, it will still be saved by. - */ -static inline void __cpu_disable_lazy_restore(unsigned int cpu) -{ - per_cpu(fpu_fpregs_owner_ctx, cpu) = NULL; -} - -static inline int fpu_want_lazy_restore(struct fpu *fpu, unsigned int cpu) -{ - return fpu == this_cpu_read_stable(fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu; -} - -static inline int is_ia32_compat_frame(void) -{ - return config_enabled(CONFIG_IA32_EMULATION) && - test_thread_flag(TIF_IA32); -} - -static inline int is_ia32_frame(void) -{ - return config_enabled(CONFIG_X86_32) || is_ia32_compat_frame(); -} - -static inline int is_x32_frame(void) -{ - return config_enabled(CONFIG_X86_X32_ABI) && test_thread_flag(TIF_X32); -} - -#define X87_FSW_ES (1 << 7) /* Exception Summary */ - -static __always_inline __pure bool use_eager_fpu(void) -{ - return static_cpu_has_safe(X86_FEATURE_EAGER_FPU); -} - -static __always_inline __pure bool use_xsaveopt(void) -{ - return static_cpu_has_safe(X86_FEATURE_XSAVEOPT); -} - -static __always_inline __pure bool use_xsave(void) -{ - return static_cpu_has_safe(X86_FEATURE_XSAVE); -} - -static __always_inline __pure bool use_fxsr(void) -{ - return static_cpu_has_safe(X86_FEATURE_FXSR); -} - -static inline void fx_finit(struct i387_fxsave_struct *fx) -{ - fx->cwd = 0x37f; - fx->mxcsr = MXCSR_DEFAULT; -} - -extern void __sanitize_i387_state(struct task_struct *); - -static inline void sanitize_i387_state(struct task_struct *tsk) -{ - if (!use_xsaveopt()) - return; - __sanitize_i387_state(tsk); -} - -#define user_insn(insn, output, input...) \ -({ \ - int err; \ - asm volatile(ASM_STAC "\n" \ - "1:" #insn "\n\t" \ - "2: " ASM_CLAC "\n" \ - ".section .fixup,\"ax\"\n" \ - "3: movl $-1,%[err]\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : [err] "=r" (err), output \ - : "0"(0), input); \ - err; \ -}) - -#define check_insn(insn, output, input...) \ -({ \ - int err; \ - asm volatile("1:" #insn "\n\t" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: movl $-1,%[err]\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : [err] "=r" (err), output \ - : "0"(0), input); \ - err; \ -}) - -static inline int fsave_user(struct i387_fsave_struct __user *fx) -{ - return user_insn(fnsave %[fx]; fwait, [fx] "=m" (*fx), "m" (*fx)); -} - -static inline int fxsave_user(struct i387_fxsave_struct __user *fx) -{ - if (config_enabled(CONFIG_X86_32)) - return user_insn(fxsave %[fx], [fx] "=m" (*fx), "m" (*fx)); - else if (config_enabled(CONFIG_AS_FXSAVEQ)) - return user_insn(fxsaveq %[fx], [fx] "=m" (*fx), "m" (*fx)); - - /* See comment in fpu_fxsave() below. */ - return user_insn(rex64/fxsave (%[fx]), "=m" (*fx), [fx] "R" (fx)); -} - -static inline int fxrstor_checking(struct i387_fxsave_struct *fx) -{ - if (config_enabled(CONFIG_X86_32)) - return check_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx)); - else if (config_enabled(CONFIG_AS_FXSAVEQ)) - return check_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx)); - - /* See comment in fpu_fxsave() below. */ - return check_insn(rex64/fxrstor (%[fx]), "=m" (*fx), [fx] "R" (fx), - "m" (*fx)); -} - -static inline int fxrstor_user(struct i387_fxsave_struct __user *fx) -{ - if (config_enabled(CONFIG_X86_32)) - return user_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx)); - else if (config_enabled(CONFIG_AS_FXSAVEQ)) - return user_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx)); - - /* See comment in fpu_fxsave() below. */ - return user_insn(rex64/fxrstor (%[fx]), "=m" (*fx), [fx] "R" (fx), - "m" (*fx)); -} - -static inline int frstor_checking(struct i387_fsave_struct *fx) -{ - return check_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline int frstor_user(struct i387_fsave_struct __user *fx) -{ - return user_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx)); -} - -static inline void fpu_fxsave(struct fpu *fpu) -{ - if (config_enabled(CONFIG_X86_32)) - asm volatile( "fxsave %[fx]" : [fx] "=m" (fpu->state->fxsave)); - else if (config_enabled(CONFIG_AS_FXSAVEQ)) - asm volatile("fxsaveq %[fx]" : [fx] "=m" (fpu->state->fxsave)); - else { - /* Using "rex64; fxsave %0" is broken because, if the memory - * operand uses any extended registers for addressing, a second - * REX prefix will be generated (to the assembler, rex64 - * followed by semicolon is a separate instruction), and hence - * the 64-bitness is lost. - * - * Using "fxsaveq %0" would be the ideal choice, but is only - * supported starting with gas 2.16. - * - * Using, as a workaround, the properly prefixed form below - * isn't accepted by any binutils version so far released, - * complaining that the same type of prefix is used twice if - * an extended register is needed for addressing (fix submitted - * to mainline 2005-11-21). - * - * asm volatile("rex64/fxsave %0" : "=m" (fpu->state->fxsave)); - * - * This, however, we can work around by forcing the compiler to - * select an addressing mode that doesn't require extended - * registers. - */ - asm volatile( "rex64/fxsave (%[fx])" - : "=m" (fpu->state->fxsave) - : [fx] "R" (&fpu->state->fxsave)); - } -} - -/* - * These must be called with preempt disabled. Returns - * 'true' if the FPU state is still intact. - */ -static inline int fpu_save_init(struct fpu *fpu) -{ - if (use_xsave()) { - xsave_state(&fpu->state->xsave); - - /* - * xsave header may indicate the init state of the FP. - */ - if (!(fpu->state->xsave.xsave_hdr.xstate_bv & XSTATE_FP)) - return 1; - } else if (use_fxsr()) { - fpu_fxsave(fpu); - } else { - asm volatile("fnsave %[fx]; fwait" - : [fx] "=m" (fpu->state->fsave)); - return 0; - } - - /* - * If exceptions are pending, we need to clear them so - * that we don't randomly get exceptions later. - * - * FIXME! Is this perhaps only true for the old-style - * irq13 case? Maybe we could leave the x87 state - * intact otherwise? - */ - if (unlikely(fpu->state->fxsave.swd & X87_FSW_ES)) { - asm volatile("fnclex"); - return 0; - } - return 1; -} - -static inline int fpu_restore_checking(struct fpu *fpu) -{ - if (use_xsave()) - return fpu_xrstor_checking(&fpu->state->xsave); - else if (use_fxsr()) - return fxrstor_checking(&fpu->state->fxsave); - else - return frstor_checking(&fpu->state->fsave); -} - -static inline int restore_fpu_checking(struct fpu *fpu) -{ - /* - * AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception is - * pending. Clear the x87 state here by setting it to fixed values. - * "m" is a random variable that should be in L1. - */ - if (unlikely(static_cpu_has_bug_safe(X86_BUG_FXSAVE_LEAK))) { - asm volatile( - "fnclex\n\t" - "emms\n\t" - "fildl %P[addr]" /* set F?P to defined value */ - : : [addr] "m" (fpu->has_fpu)); - } - - return fpu_restore_checking(fpu); -} - -/* Must be paired with an 'stts' after! */ -static inline void __thread_clear_has_fpu(struct fpu *fpu) -{ - fpu->has_fpu = 0; - this_cpu_write(fpu_fpregs_owner_ctx, NULL); -} - -/* Must be paired with a 'clts' before! */ -static inline void __thread_set_has_fpu(struct fpu *fpu) -{ - fpu->has_fpu = 1; - this_cpu_write(fpu_fpregs_owner_ctx, fpu); -} - -/* - * Encapsulate the CR0.TS handling together with the - * software flag. - * - * These generally need preemption protection to work, - * do try to avoid using these on their own. - */ -static inline void __thread_fpu_end(struct fpu *fpu) -{ - __thread_clear_has_fpu(fpu); - if (!use_eager_fpu()) - stts(); -} - -static inline void __thread_fpu_begin(struct fpu *fpu) -{ - if (!use_eager_fpu()) - clts(); - __thread_set_has_fpu(fpu); -} - -static inline void drop_fpu(struct fpu *fpu) -{ - /* - * Forget coprocessor state.. - */ - preempt_disable(); - fpu->counter = 0; - - if (fpu->has_fpu) { - /* Ignore delayed exceptions from user space */ - asm volatile("1: fwait\n" - "2:\n" - _ASM_EXTABLE(1b, 2b)); - __thread_fpu_end(fpu); - } - - fpu->fpstate_active = 0; - - preempt_enable(); -} - -static inline void restore_init_xstate(void) -{ - if (use_xsave()) - xrstor_state(init_xstate_buf, -1); - else - fxrstor_checking(&init_xstate_buf->i387); -} - -/* - * Reset the FPU state in the eager case and drop it in the lazy case (later use - * will reinit it). - */ -static inline void fpu_reset_state(struct fpu *fpu) -{ - if (!use_eager_fpu()) - drop_fpu(fpu); - else - restore_init_xstate(); -} - -/* - * FPU state switching for scheduling. - * - * This is a two-stage process: - * - * - switch_fpu_prepare() saves the old state and - * sets the new state of the CR0.TS bit. This is - * done within the context of the old process. - * - * - switch_fpu_finish() restores the new state as - * necessary. - */ -typedef struct { int preload; } fpu_switch_t; - -static inline fpu_switch_t -switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu) -{ - fpu_switch_t fpu; - - /* - * If the task has used the math, pre-load the FPU on xsave processors - * or if the past 5 consecutive context-switches used math. - */ - fpu.preload = new_fpu->fpstate_active && - (use_eager_fpu() || new_fpu->counter > 5); - - if (old_fpu->has_fpu) { - if (!fpu_save_init(old_fpu)) - old_fpu->last_cpu = -1; - else - old_fpu->last_cpu = cpu; - - /* But leave fpu_fpregs_owner_ctx! */ - old_fpu->has_fpu = 0; - - /* Don't change CR0.TS if we just switch! */ - if (fpu.preload) { - new_fpu->counter++; - __thread_set_has_fpu(new_fpu); - prefetch(new_fpu->state); - } else if (!use_eager_fpu()) - stts(); - } else { - old_fpu->counter = 0; - old_fpu->last_cpu = -1; - if (fpu.preload) { - new_fpu->counter++; - if (fpu_want_lazy_restore(new_fpu, cpu)) - fpu.preload = 0; - else - prefetch(new_fpu->state); - __thread_fpu_begin(new_fpu); - } - } - return fpu; -} - -/* - * By the time this gets called, we've already cleared CR0.TS and - * given the process the FPU if we are going to preload the FPU - * state - all we need to do is to conditionally restore the register - * state itself. - */ -static inline void switch_fpu_finish(struct fpu *new_fpu, fpu_switch_t fpu_switch) -{ - if (fpu_switch.preload) { - if (unlikely(restore_fpu_checking(new_fpu))) - fpu_reset_state(new_fpu); - } -} - -/* - * Signal frame handlers... - */ -extern int save_xstate_sig(void __user *buf, void __user *fx, int size); -extern int __restore_xstate_sig(void __user *buf, void __user *fx, int size); - -static inline int xstate_sigframe_size(void) -{ - return use_xsave() ? xstate_size + FP_XSTATE_MAGIC2_SIZE : xstate_size; -} - -static inline int restore_xstate_sig(void __user *buf, int ia32_frame) -{ - void __user *buf_fx = buf; - int size = xstate_sigframe_size(); - - if (ia32_frame && use_fxsr()) { - buf_fx = buf + sizeof(struct i387_fsave_struct); - size += sizeof(struct i387_fsave_struct); - } - - return __restore_xstate_sig(buf, buf_fx, size); -} - -/* - * Needs to be preemption-safe. - * - * NOTE! user_fpu_begin() must be used only immediately before restoring - * the save state. It does not do any saving/restoring on its own. In - * lazy FPU mode, it is just an optimization to avoid a #NM exception, - * the task can lose the FPU right after preempt_enable(). - */ -static inline void user_fpu_begin(void) -{ - struct fpu *fpu = ¤t->thread.fpu; - - preempt_disable(); - if (!user_has_fpu()) - __thread_fpu_begin(fpu); - preempt_enable(); -} - -/* - * i387 state interaction - */ -static inline unsigned short get_fpu_cwd(struct task_struct *tsk) -{ - if (cpu_has_fxsr) { - return tsk->thread.fpu.state->fxsave.cwd; - } else { - return (unsigned short)tsk->thread.fpu.state->fsave.cwd; - } -} - -static inline unsigned short get_fpu_swd(struct task_struct *tsk) -{ - if (cpu_has_fxsr) { - return tsk->thread.fpu.state->fxsave.swd; - } else { - return (unsigned short)tsk->thread.fpu.state->fsave.swd; - } -} - -static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) -{ - if (cpu_has_xmm) { - return tsk->thread.fpu.state->fxsave.mxcsr; - } else { - return MXCSR_DEFAULT; - } -} - -extern void fpstate_cache_init(void); - -extern int fpstate_alloc(struct fpu *fpu); -extern void fpstate_free(struct fpu *fpu); -extern int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu); - -static inline unsigned long -alloc_mathframe(unsigned long sp, int ia32_frame, unsigned long *buf_fx, - unsigned long *size) -{ - unsigned long frame_size = xstate_sigframe_size(); - - *buf_fx = sp = round_down(sp - frame_size, 64); - if (ia32_frame && use_fxsr()) { - frame_size += sizeof(struct i387_fsave_struct); - sp -= sizeof(struct i387_fsave_struct); - } - - *size = frame_size; - return sp; -} - -#endif diff --git a/arch/x86/include/asm/fpu/internal.h b/arch/x86/include/asm/fpu/internal.h new file mode 100644 index 000000000000..386a8837c358 --- /dev/null +++ b/arch/x86/include/asm/fpu/internal.h @@ -0,0 +1,556 @@ +/* + * Copyright (C) 1994 Linus Torvalds + * + * Pentium III FXSR, SSE support + * General FPU state handling cleanups + * Gareth Hughes , May 2000 + * x86-64 work by Andi Kleen 2002 + */ + +#ifndef _ASM_X86_FPU_INTERNAL_H +#define _ASM_X86_FPU_INTERNAL_H + +#include +#include +#include + +#include +#include +#include + +#ifdef CONFIG_X86_64 +# include +# include +struct ksignal; +int ia32_setup_rt_frame(int sig, struct ksignal *ksig, + compat_sigset_t *set, struct pt_regs *regs); +int ia32_setup_frame(int sig, struct ksignal *ksig, + compat_sigset_t *set, struct pt_regs *regs); +#else +# define user_i387_ia32_struct user_i387_struct +# define user32_fxsr_struct user_fxsr_struct +# define ia32_setup_frame __setup_frame +# define ia32_setup_rt_frame __setup_rt_frame +#endif + +extern unsigned int mxcsr_feature_mask; +extern void fpu__cpu_init(void); +extern void eager_fpu_init(void); + +DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx); + +extern void convert_from_fxsr(struct user_i387_ia32_struct *env, + struct task_struct *tsk); +extern void convert_to_fxsr(struct task_struct *tsk, + const struct user_i387_ia32_struct *env); + +extern user_regset_active_fn fpregs_active, xfpregs_active; +extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get, + xstateregs_get; +extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set, + xstateregs_set; + +/* + * xstateregs_active == fpregs_active. Please refer to the comment + * at the definition of fpregs_active. + */ +#define xstateregs_active fpregs_active + +#ifdef CONFIG_MATH_EMULATION +extern void finit_soft_fpu(struct i387_soft_struct *soft); +#else +static inline void finit_soft_fpu(struct i387_soft_struct *soft) {} +#endif + +/* + * Must be run with preemption disabled: this clears the fpu_fpregs_owner_ctx, + * on this CPU. + * + * This will disable any lazy FPU state restore of the current FPU state, + * but if the current thread owns the FPU, it will still be saved by. + */ +static inline void __cpu_disable_lazy_restore(unsigned int cpu) +{ + per_cpu(fpu_fpregs_owner_ctx, cpu) = NULL; +} + +static inline int fpu_want_lazy_restore(struct fpu *fpu, unsigned int cpu) +{ + return fpu == this_cpu_read_stable(fpu_fpregs_owner_ctx) && cpu == fpu->last_cpu; +} + +static inline int is_ia32_compat_frame(void) +{ + return config_enabled(CONFIG_IA32_EMULATION) && + test_thread_flag(TIF_IA32); +} + +static inline int is_ia32_frame(void) +{ + return config_enabled(CONFIG_X86_32) || is_ia32_compat_frame(); +} + +static inline int is_x32_frame(void) +{ + return config_enabled(CONFIG_X86_X32_ABI) && test_thread_flag(TIF_X32); +} + +#define X87_FSW_ES (1 << 7) /* Exception Summary */ + +static __always_inline __pure bool use_eager_fpu(void) +{ + return static_cpu_has_safe(X86_FEATURE_EAGER_FPU); +} + +static __always_inline __pure bool use_xsaveopt(void) +{ + return static_cpu_has_safe(X86_FEATURE_XSAVEOPT); +} + +static __always_inline __pure bool use_xsave(void) +{ + return static_cpu_has_safe(X86_FEATURE_XSAVE); +} + +static __always_inline __pure bool use_fxsr(void) +{ + return static_cpu_has_safe(X86_FEATURE_FXSR); +} + +static inline void fx_finit(struct i387_fxsave_struct *fx) +{ + fx->cwd = 0x37f; + fx->mxcsr = MXCSR_DEFAULT; +} + +extern void __sanitize_i387_state(struct task_struct *); + +static inline void sanitize_i387_state(struct task_struct *tsk) +{ + if (!use_xsaveopt()) + return; + __sanitize_i387_state(tsk); +} + +#define user_insn(insn, output, input...) \ +({ \ + int err; \ + asm volatile(ASM_STAC "\n" \ + "1:" #insn "\n\t" \ + "2: " ASM_CLAC "\n" \ + ".section .fixup,\"ax\"\n" \ + "3: movl $-1,%[err]\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : [err] "=r" (err), output \ + : "0"(0), input); \ + err; \ +}) + +#define check_insn(insn, output, input...) \ +({ \ + int err; \ + asm volatile("1:" #insn "\n\t" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: movl $-1,%[err]\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE(1b, 3b) \ + : [err] "=r" (err), output \ + : "0"(0), input); \ + err; \ +}) + +static inline int fsave_user(struct i387_fsave_struct __user *fx) +{ + return user_insn(fnsave %[fx]; fwait, [fx] "=m" (*fx), "m" (*fx)); +} + +static inline int fxsave_user(struct i387_fxsave_struct __user *fx) +{ + if (config_enabled(CONFIG_X86_32)) + return user_insn(fxsave %[fx], [fx] "=m" (*fx), "m" (*fx)); + else if (config_enabled(CONFIG_AS_FXSAVEQ)) + return user_insn(fxsaveq %[fx], [fx] "=m" (*fx), "m" (*fx)); + + /* See comment in fpu_fxsave() below. */ + return user_insn(rex64/fxsave (%[fx]), "=m" (*fx), [fx] "R" (fx)); +} + +static inline int fxrstor_checking(struct i387_fxsave_struct *fx) +{ + if (config_enabled(CONFIG_X86_32)) + return check_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx)); + else if (config_enabled(CONFIG_AS_FXSAVEQ)) + return check_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx)); + + /* See comment in fpu_fxsave() below. */ + return check_insn(rex64/fxrstor (%[fx]), "=m" (*fx), [fx] "R" (fx), + "m" (*fx)); +} + +static inline int fxrstor_user(struct i387_fxsave_struct __user *fx) +{ + if (config_enabled(CONFIG_X86_32)) + return user_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx)); + else if (config_enabled(CONFIG_AS_FXSAVEQ)) + return user_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx)); + + /* See comment in fpu_fxsave() below. */ + return user_insn(rex64/fxrstor (%[fx]), "=m" (*fx), [fx] "R" (fx), + "m" (*fx)); +} + +static inline int frstor_checking(struct i387_fsave_struct *fx) +{ + return check_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx)); +} + +static inline int frstor_user(struct i387_fsave_struct __user *fx) +{ + return user_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx)); +} + +static inline void fpu_fxsave(struct fpu *fpu) +{ + if (config_enabled(CONFIG_X86_32)) + asm volatile( "fxsave %[fx]" : [fx] "=m" (fpu->state->fxsave)); + else if (config_enabled(CONFIG_AS_FXSAVEQ)) + asm volatile("fxsaveq %[fx]" : [fx] "=m" (fpu->state->fxsave)); + else { + /* Using "rex64; fxsave %0" is broken because, if the memory + * operand uses any extended registers for addressing, a second + * REX prefix will be generated (to the assembler, rex64 + * followed by semicolon is a separate instruction), and hence + * the 64-bitness is lost. + * + * Using "fxsaveq %0" would be the ideal choice, but is only + * supported starting with gas 2.16. + * + * Using, as a workaround, the properly prefixed form below + * isn't accepted by any binutils version so far released, + * complaining that the same type of prefix is used twice if + * an extended register is needed for addressing (fix submitted + * to mainline 2005-11-21). + * + * asm volatile("rex64/fxsave %0" : "=m" (fpu->state->fxsave)); + * + * This, however, we can work around by forcing the compiler to + * select an addressing mode that doesn't require extended + * registers. + */ + asm volatile( "rex64/fxsave (%[fx])" + : "=m" (fpu->state->fxsave) + : [fx] "R" (&fpu->state->fxsave)); + } +} + +/* + * These must be called with preempt disabled. Returns + * 'true' if the FPU state is still intact. + */ +static inline int fpu_save_init(struct fpu *fpu) +{ + if (use_xsave()) { + xsave_state(&fpu->state->xsave); + + /* + * xsave header may indicate the init state of the FP. + */ + if (!(fpu->state->xsave.xsave_hdr.xstate_bv & XSTATE_FP)) + return 1; + } else if (use_fxsr()) { + fpu_fxsave(fpu); + } else { + asm volatile("fnsave %[fx]; fwait" + : [fx] "=m" (fpu->state->fsave)); + return 0; + } + + /* + * If exceptions are pending, we need to clear them so + * that we don't randomly get exceptions later. + * + * FIXME! Is this perhaps only true for the old-style + * irq13 case? Maybe we could leave the x87 state + * intact otherwise? + */ + if (unlikely(fpu->state->fxsave.swd & X87_FSW_ES)) { + asm volatile("fnclex"); + return 0; + } + return 1; +} + +static inline int fpu_restore_checking(struct fpu *fpu) +{ + if (use_xsave()) + return fpu_xrstor_checking(&fpu->state->xsave); + else if (use_fxsr()) + return fxrstor_checking(&fpu->state->fxsave); + else + return frstor_checking(&fpu->state->fsave); +} + +static inline int restore_fpu_checking(struct fpu *fpu) +{ + /* + * AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception is + * pending. Clear the x87 state here by setting it to fixed values. + * "m" is a random variable that should be in L1. + */ + if (unlikely(static_cpu_has_bug_safe(X86_BUG_FXSAVE_LEAK))) { + asm volatile( + "fnclex\n\t" + "emms\n\t" + "fildl %P[addr]" /* set F?P to defined value */ + : : [addr] "m" (fpu->has_fpu)); + } + + return fpu_restore_checking(fpu); +} + +/* Must be paired with an 'stts' after! */ +static inline void __thread_clear_has_fpu(struct fpu *fpu) +{ + fpu->has_fpu = 0; + this_cpu_write(fpu_fpregs_owner_ctx, NULL); +} + +/* Must be paired with a 'clts' before! */ +static inline void __thread_set_has_fpu(struct fpu *fpu) +{ + fpu->has_fpu = 1; + this_cpu_write(fpu_fpregs_owner_ctx, fpu); +} + +/* + * Encapsulate the CR0.TS handling together with the + * software flag. + * + * These generally need preemption protection to work, + * do try to avoid using these on their own. + */ +static inline void __thread_fpu_end(struct fpu *fpu) +{ + __thread_clear_has_fpu(fpu); + if (!use_eager_fpu()) + stts(); +} + +static inline void __thread_fpu_begin(struct fpu *fpu) +{ + if (!use_eager_fpu()) + clts(); + __thread_set_has_fpu(fpu); +} + +static inline void drop_fpu(struct fpu *fpu) +{ + /* + * Forget coprocessor state.. + */ + preempt_disable(); + fpu->counter = 0; + + if (fpu->has_fpu) { + /* Ignore delayed exceptions from user space */ + asm volatile("1: fwait\n" + "2:\n" + _ASM_EXTABLE(1b, 2b)); + __thread_fpu_end(fpu); + } + + fpu->fpstate_active = 0; + + preempt_enable(); +} + +static inline void restore_init_xstate(void) +{ + if (use_xsave()) + xrstor_state(init_xstate_buf, -1); + else + fxrstor_checking(&init_xstate_buf->i387); +} + +/* + * Reset the FPU state in the eager case and drop it in the lazy case (later use + * will reinit it). + */ +static inline void fpu_reset_state(struct fpu *fpu) +{ + if (!use_eager_fpu()) + drop_fpu(fpu); + else + restore_init_xstate(); +} + +/* + * FPU state switching for scheduling. + * + * This is a two-stage process: + * + * - switch_fpu_prepare() saves the old state and + * sets the new state of the CR0.TS bit. This is + * done within the context of the old process. + * + * - switch_fpu_finish() restores the new state as + * necessary. + */ +typedef struct { int preload; } fpu_switch_t; + +static inline fpu_switch_t +switch_fpu_prepare(struct fpu *old_fpu, struct fpu *new_fpu, int cpu) +{ + fpu_switch_t fpu; + + /* + * If the task has used the math, pre-load the FPU on xsave processors + * or if the past 5 consecutive context-switches used math. + */ + fpu.preload = new_fpu->fpstate_active && + (use_eager_fpu() || new_fpu->counter > 5); + + if (old_fpu->has_fpu) { + if (!fpu_save_init(old_fpu)) + old_fpu->last_cpu = -1; + else + old_fpu->last_cpu = cpu; + + /* But leave fpu_fpregs_owner_ctx! */ + old_fpu->has_fpu = 0; + + /* Don't change CR0.TS if we just switch! */ + if (fpu.preload) { + new_fpu->counter++; + __thread_set_has_fpu(new_fpu); + prefetch(new_fpu->state); + } else if (!use_eager_fpu()) + stts(); + } else { + old_fpu->counter = 0; + old_fpu->last_cpu = -1; + if (fpu.preload) { + new_fpu->counter++; + if (fpu_want_lazy_restore(new_fpu, cpu)) + fpu.preload = 0; + else + prefetch(new_fpu->state); + __thread_fpu_begin(new_fpu); + } + } + return fpu; +} + +/* + * By the time this gets called, we've already cleared CR0.TS and + * given the process the FPU if we are going to preload the FPU + * state - all we need to do is to conditionally restore the register + * state itself. + */ +static inline void switch_fpu_finish(struct fpu *new_fpu, fpu_switch_t fpu_switch) +{ + if (fpu_switch.preload) { + if (unlikely(restore_fpu_checking(new_fpu))) + fpu_reset_state(new_fpu); + } +} + +/* + * Signal frame handlers... + */ +extern int save_xstate_sig(void __user *buf, void __user *fx, int size); +extern int __restore_xstate_sig(void __user *buf, void __user *fx, int size); + +static inline int xstate_sigframe_size(void) +{ + return use_xsave() ? xstate_size + FP_XSTATE_MAGIC2_SIZE : xstate_size; +} + +static inline int restore_xstate_sig(void __user *buf, int ia32_frame) +{ + void __user *buf_fx = buf; + int size = xstate_sigframe_size(); + + if (ia32_frame && use_fxsr()) { + buf_fx = buf + sizeof(struct i387_fsave_struct); + size += sizeof(struct i387_fsave_struct); + } + + return __restore_xstate_sig(buf, buf_fx, size); +} + +/* + * Needs to be preemption-safe. + * + * NOTE! user_fpu_begin() must be used only immediately before restoring + * the save state. It does not do any saving/restoring on its own. In + * lazy FPU mode, it is just an optimization to avoid a #NM exception, + * the task can lose the FPU right after preempt_enable(). + */ +static inline void user_fpu_begin(void) +{ + struct fpu *fpu = ¤t->thread.fpu; + + preempt_disable(); + if (!user_has_fpu()) + __thread_fpu_begin(fpu); + preempt_enable(); +} + +/* + * i387 state interaction + */ +static inline unsigned short get_fpu_cwd(struct task_struct *tsk) +{ + if (cpu_has_fxsr) { + return tsk->thread.fpu.state->fxsave.cwd; + } else { + return (unsigned short)tsk->thread.fpu.state->fsave.cwd; + } +} + +static inline unsigned short get_fpu_swd(struct task_struct *tsk) +{ + if (cpu_has_fxsr) { + return tsk->thread.fpu.state->fxsave.swd; + } else { + return (unsigned short)tsk->thread.fpu.state->fsave.swd; + } +} + +static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) +{ + if (cpu_has_xmm) { + return tsk->thread.fpu.state->fxsave.mxcsr; + } else { + return MXCSR_DEFAULT; + } +} + +extern void fpstate_cache_init(void); + +extern int fpstate_alloc(struct fpu *fpu); +extern void fpstate_free(struct fpu *fpu); +extern int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu); + +static inline unsigned long +alloc_mathframe(unsigned long sp, int ia32_frame, unsigned long *buf_fx, + unsigned long *size) +{ + unsigned long frame_size = xstate_sigframe_size(); + + *buf_fx = sp = round_down(sp - frame_size, 64); + if (ia32_frame && use_fxsr()) { + frame_size += sizeof(struct i387_fsave_struct); + sp -= sizeof(struct i387_fsave_struct); + } + + *size = frame_size; + return sp; +} + +#endif /* _ASM_X86_FPU_INTERNAL_H */ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 88bb7a75f5c6..8f6a4ea39657 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 176f69b24358..30016f03f25d 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -5,7 +5,7 @@ * General FPU state handling cleanups * Gareth Hughes , May 2000 */ -#include +#include /* * Track whether the kernel is using the FPU state diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c index 4eabb426e910..33df056b1624 100644 --- a/arch/x86/kernel/fpu/init.c +++ b/arch/x86/kernel/fpu/init.c @@ -1,7 +1,7 @@ /* * x86 FPU boot time init code */ -#include +#include #include /* diff --git a/arch/x86/kernel/fpu/xsave.c b/arch/x86/kernel/fpu/xsave.c index 8aa3b864a2e0..4ff726e4e29b 100644 --- a/arch/x86/kernel/fpu/xsave.c +++ b/arch/x86/kernel/fpu/xsave.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 51ad3422e728..71f4b4d2f1fd 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c index 7adc314b5075..deff651835b4 100644 --- a/arch/x86/kernel/process_32.c +++ b/arch/x86/kernel/process_32.c @@ -39,7 +39,7 @@ #include #include #include -#include +#include #include #ifdef CONFIG_MATH_EMULATION #include diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index 4504569c6c4e..c50e013b57d2 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -38,7 +38,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c index 69451b8965f7..c14a00f54b61 100644 --- a/arch/x86/kernel/ptrace.c +++ b/arch/x86/kernel/ptrace.c @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c index bcb853e44d30..c67f96c87938 100644 --- a/arch/x86/kernel/signal.c +++ b/arch/x86/kernel/signal.c @@ -26,7 +26,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 60e331ceb844..29f105f0d9fb 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -68,7 +68,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 8abcd6a6f3dc..a65586edbb57 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -54,7 +54,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 479d4ce25081..91d7f3b1e50c 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -60,7 +60,7 @@ #include #include #include -#include /* Ugh! */ +#include /* Ugh! */ #include #include #include diff --git a/arch/x86/mm/mpx.c b/arch/x86/mm/mpx.c index 412b5f81e547..5563be313fd6 100644 --- a/arch/x86/mm/mpx.c +++ b/arch/x86/mm/mpx.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include static const char *mpx_mapping_name(struct vm_area_struct *vma) { diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c index 757678fb26e1..edaf934c749e 100644 --- a/arch/x86/power/cpu.c +++ b/arch/x86/power/cpu.c @@ -21,7 +21,7 @@ #include #include #include -#include /* pcntxt_mask */ +#include /* pcntxt_mask */ #include #ifdef CONFIG_X86_32 -- cgit v1.2.3 From c4d6ee6e2e52ec604cc1d76877791f8e8f5c79b5 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Mon, 27 Apr 2015 05:52:40 +0200 Subject: x86/fpu: Remove failure paths from fpstate-alloc low level functions Now that we always allocate the FPU context as part of task_struct there's no need for separate allocations - remove them and their primary failure handling code. ( Note that there's still secondary error codes that have become superfluous, those will be removed in separate patches. ) Move the somewhat misplaced setup_xstate_comp() call to the core. Reviewed-by: Borislav Petkov Cc: Andy Lutomirski Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/include/asm/fpu/internal.h | 4 --- arch/x86/kernel/fpu/core.c | 51 ++----------------------------------- arch/x86/kernel/fpu/init.c | 1 + arch/x86/kernel/process.c | 10 -------- arch/x86/kvm/x86.c | 11 -------- 5 files changed, 3 insertions(+), 74 deletions(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/include/asm/fpu/internal.h b/arch/x86/include/asm/fpu/internal.h index 4ce830fb3f31..9454f21f0edf 100644 --- a/arch/x86/include/asm/fpu/internal.h +++ b/arch/x86/include/asm/fpu/internal.h @@ -558,10 +558,6 @@ static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) } } -extern void fpstate_cache_init(void); - -extern int fpstate_alloc(struct fpu *fpu); -extern void fpstate_free(struct fpu *fpu); extern int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu); static inline unsigned long diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 1697a9a34ff0..6b8d3e1b6ef8 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -225,34 +225,6 @@ void fpstate_init(struct fpu *fpu) } EXPORT_SYMBOL_GPL(fpstate_init); -/* - * FPU state allocation: - */ -static struct kmem_cache *task_xstate_cachep; - -void fpstate_cache_init(void) -{ - task_xstate_cachep = - kmem_cache_create("task_xstate", xstate_size, - __alignof__(union thread_xstate), - SLAB_PANIC | SLAB_NOTRACK, NULL); - setup_xstate_comp(); -} - -int fpstate_alloc(struct fpu *fpu) -{ - /* The CPU requires the FPU state to be aligned to 16 byte boundaries: */ - WARN_ON((unsigned long)&fpu->state & 15); - - return 0; -} -EXPORT_SYMBOL_GPL(fpstate_alloc); - -void fpstate_free(struct fpu *fpu) -{ -} -EXPORT_SYMBOL_GPL(fpstate_free); - /* * Copy the current task's FPU state to a new task's FPU context. * @@ -280,13 +252,9 @@ int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu) dst_fpu->fpregs_active = 0; dst_fpu->last_cpu = -1; - if (src_fpu->fpstate_active) { - int err = fpstate_alloc(dst_fpu); - - if (err) - return err; + if (src_fpu->fpstate_active) fpu_copy(dst_fpu, src_fpu); - } + return 0; } @@ -305,13 +273,6 @@ int fpstate_alloc_init(struct fpu *fpu) if (WARN_ON_ONCE(fpu->fpstate_active)) return -EINVAL; - /* - * Memory allocation at the first usage of the FPU and other state. - */ - ret = fpstate_alloc(fpu); - if (ret) - return ret; - fpstate_init(fpu); /* Safe to do for the current task: */ @@ -356,13 +317,6 @@ static int fpu__unlazy_stopped(struct fpu *child_fpu) return 0; } - /* - * Memory allocation at the first usage of the FPU and other state. - */ - ret = fpstate_alloc(child_fpu); - if (ret) - return ret; - fpstate_init(child_fpu); /* Safe to do for stopped child tasks: */ @@ -423,7 +377,6 @@ void fpu__clear(struct task_struct *tsk) if (!use_eager_fpu()) { /* FPU state will be reallocated lazily at the first use. */ drop_fpu(fpu); - fpstate_free(fpu); } else { if (!fpu->fpstate_active) { /* kthread execs. TODO: cleanup this horror. */ diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c index 7ae5a62918c7..460e7e2c6186 100644 --- a/arch/x86/kernel/fpu/init.c +++ b/arch/x86/kernel/fpu/init.c @@ -265,6 +265,7 @@ void fpu__init_system(struct cpuinfo_x86 *c) fpu__init_system_generic(); fpu__init_system_xstate_size_legacy(); fpu__init_system_xstate(); + setup_xstate_comp(); fpu__init_system_ctx_switch(); } diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 71f4b4d2f1fd..5d37c26fa89f 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -86,16 +86,6 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) return fpu__copy(&dst->thread.fpu, &src->thread.fpu); } -void arch_release_task_struct(struct task_struct *tsk) -{ - fpstate_free(&tsk->thread.fpu); -} - -void arch_task_cache_init(void) -{ - fpstate_cache_init(); -} - /* * Free current thread data structures etc.. */ diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 8bb0de5bf9c0..68529251e897 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7008,10 +7008,6 @@ int fx_init(struct kvm_vcpu *vcpu) { int err; - err = fpstate_alloc(&vcpu->arch.guest_fpu); - if (err) - return err; - fpstate_init(&vcpu->arch.guest_fpu); if (cpu_has_xsaves) vcpu->arch.guest_fpu.state.xsave.header.xcomp_bv = @@ -7028,11 +7024,6 @@ int fx_init(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(fx_init); -static void fx_free(struct kvm_vcpu *vcpu) -{ - fpstate_free(&vcpu->arch.guest_fpu); -} - void kvm_load_guest_fpu(struct kvm_vcpu *vcpu) { if (vcpu->guest_fpu_loaded) @@ -7070,7 +7061,6 @@ void kvm_arch_vcpu_free(struct kvm_vcpu *vcpu) kvmclock_reset(vcpu); free_cpumask_var(vcpu->arch.wbinvd_dirty_mask); - fx_free(vcpu); kvm_x86_ops->vcpu_free(vcpu); } @@ -7126,7 +7116,6 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) kvm_mmu_unload(vcpu); vcpu_put(vcpu); - fx_free(vcpu); kvm_x86_ops->vcpu_free(vcpu); } -- cgit v1.2.3 From 5033861575df08a04090cc7b785b2b7aadcbde82 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 29 Apr 2015 19:04:31 +0200 Subject: x86/fpu: Synchronize the naming of drop_fpu() and fpu_reset_state() drop_fpu() and fpu_reset_state() are similar in functionality and in scope, yet this is not apparent from their names. drop_fpu() deactivates FPU contents (both the fpregs and the fpstate), but leaves register contents intact in the eager-FPU case, mostly as an optimization. It disables fpregs in the lazy FPU case. The drop_fpu() method can be used to destroy FPU state in an optimized way, when we know that a new state will be loaded before user-space might see any remains of the old FPU state: - such as in sys_exit()'s exit_thread() where we know this task won't execute any user-space instructions anymore and the next context switch cleans up the FPU. The old FPU state might still be around in the eagerfpu case but won't be saved. - in __restore_xstate_sig(), where we use drop_fpu() before copying a new state into the fpstate and activating that one. No user-pace instructions can execute between those steps. - in sys_execve()'s fpu__clear(): there we use drop_fpu() in the !eagerfpu case, where it's equivalent to a full reinit. fpu_reset_state() is a stronger version of drop_fpu(): both in the eagerfpu and the lazy-FPU case it guarantees that fpregs are reinitialized to init state. This method is used in cases where we need a full reset: - handle_signal() uses fpu_reset_state() to reset the FPU state to init before executing a user-space signal handler. While we have already saved the original FPU state at this point, and always restore the original state, the signal handling code still has to do this reinit, because signals may interrupt any user-space instruction, and the FPU might be in various intermediate states (such as an unbalanced x87 stack) that is not immediately usable for general C signal handler code. - __restore_xstate_sig() uses fpu_reset_state() when the signal frame has no FP context. Since the signal handler may have modified the FPU state, it gets reset back to init state. - in another branch __restore_xstate_sig() uses fpu_reset_state() to handle a restoration error: when restore_user_xstate() fails to restore FPU state and we might have inconsistent FPU data, fpu_reset_state() is used to reset it back to a known good state. - __kernel_fpu_end() uses fpu_reset_state() in an error branch. This is in a 'must not trigger' error branch, so on bug-free kernels this never triggers. - fpu__restore() uses fpu_reset_state() in an error path as well: if the fpstate was set up with invalid FPU state (via ptrace or via a signal handler), then it's reset back to init state. - likewise, the scheduler's switch_fpu_finish() uses it in a restoration error path too. Move both drop_fpu() and fpu_reset_state() to the fpu__*() namespace and harmonize their naming with their function: fpu__drop() fpu__reset() This clearly shows that both methods operate on the full state of the FPU, just like fpu__restore(). Also add comments to explain what each function does. Cc: Andy Lutomirski Cc: Borislav Petkov Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/include/asm/fpu/internal.h | 23 ++++++++++++++--------- arch/x86/kernel/fpu/core.c | 6 +++--- arch/x86/kernel/fpu/xstate.c | 6 +++--- arch/x86/kernel/process.c | 2 +- arch/x86/kernel/signal.c | 2 +- 5 files changed, 22 insertions(+), 17 deletions(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/include/asm/fpu/internal.h b/arch/x86/include/asm/fpu/internal.h index 0f17cd4e4e58..31bfda818f30 100644 --- a/arch/x86/include/asm/fpu/internal.h +++ b/arch/x86/include/asm/fpu/internal.h @@ -382,11 +382,17 @@ static inline void fpregs_deactivate(struct fpu *fpu) __fpregs_deactivate_hw(); } -static inline void drop_fpu(struct fpu *fpu) +/* + * Drops current FPU state: deactivates the fpregs and + * the fpstate. NOTE: it still leaves previous contents + * in the fpregs in the eager-FPU case. + * + * This function can be used in cases where we know that + * a state-restore is coming: either an explicit one, + * or a reschedule. + */ +static inline void fpu__drop(struct fpu *fpu) { - /* - * Forget coprocessor state.. - */ preempt_disable(); fpu->counter = 0; @@ -412,13 +418,12 @@ static inline void restore_init_xstate(void) } /* - * Reset the FPU state in the eager case and drop it in the lazy case (later use - * will reinit it). + * Reset the FPU state back to init state. */ -static inline void fpu_reset_state(struct fpu *fpu) +static inline void fpu__reset(struct fpu *fpu) { if (!use_eager_fpu()) - drop_fpu(fpu); + fpu__drop(fpu); else restore_init_xstate(); } @@ -516,7 +521,7 @@ static inline void switch_fpu_finish(struct fpu *new_fpu, fpu_switch_t fpu_switc { if (fpu_switch.preload) { if (unlikely(restore_fpu_checking(new_fpu))) - fpu_reset_state(new_fpu); + fpu__reset(new_fpu); } } diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index a2e2da2b08c5..bf217cde114d 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -116,7 +116,7 @@ void __kernel_fpu_end(void) if (fpu->fpregs_active) { if (WARN_ON(restore_fpu_checking(fpu))) - fpu_reset_state(fpu); + fpu__reset(fpu); } else { __fpregs_deactivate_hw(); } @@ -339,7 +339,7 @@ void fpu__restore(void) kernel_fpu_disable(); fpregs_activate(fpu); if (unlikely(restore_fpu_checking(fpu))) { - fpu_reset_state(fpu); + fpu__reset(fpu); force_sig_info(SIGSEGV, SEND_SIG_PRIV, tsk); } else { tsk->thread.fpu.counter++; @@ -360,7 +360,7 @@ void fpu__clear(struct task_struct *tsk) if (!use_eager_fpu()) { /* FPU state will be reallocated lazily at the first use. */ - drop_fpu(fpu); + fpu__drop(fpu); } else { if (!fpu->fpstate_active) { fpu__activate_curr(fpu); diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index b8e5fee2aef3..5e3d9242bb95 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -401,7 +401,7 @@ int __restore_xstate_sig(void __user *buf, void __user *buf_fx, int size) config_enabled(CONFIG_IA32_EMULATION)); if (!buf) { - fpu_reset_state(fpu); + fpu__reset(fpu); return 0; } @@ -449,7 +449,7 @@ int __restore_xstate_sig(void __user *buf, void __user *buf_fx, int size) * We will be ready to restore/save the state only after * fpu->fpstate_active is again set. */ - drop_fpu(fpu); + fpu__drop(fpu); if (__copy_from_user(&fpu->state.xsave, buf_fx, state_size) || __copy_from_user(&env, buf, sizeof(env))) { @@ -474,7 +474,7 @@ int __restore_xstate_sig(void __user *buf, void __user *buf_fx, int size) */ user_fpu_begin(); if (restore_user_xstate(buf_fx, xfeatures, fx_only)) { - fpu_reset_state(fpu); + fpu__reset(fpu); return -1; } } diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 5d37c26fa89f..dde263fb2031 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -110,7 +110,7 @@ void exit_thread(void) kfree(bp); } - drop_fpu(fpu); + fpu__drop(fpu); } void flush_thread(void) diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c index 59cfc9c97491..6bf512390536 100644 --- a/arch/x86/kernel/signal.c +++ b/arch/x86/kernel/signal.c @@ -667,7 +667,7 @@ handle_signal(struct ksignal *ksig, struct pt_regs *regs) * Ensure the signal handler starts with the new fpu state. */ if (fpu->fpstate_active) - fpu_reset_state(fpu); + fpu__reset(fpu); } signal_setup_done(failed, ksig, stepping); } -- cgit v1.2.3 From 04c8e01d50ae1f2b33a46459e8b1e776b747e97d Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Wed, 29 Apr 2015 20:35:33 +0200 Subject: x86/fpu: Move fpu__clear() to 'struct fpu *' parameter passing Do it like all other high level FPU state handling functions: they only know about struct fpu, not about the task. (Also remove a dead prototype while at it.) Cc: Andy Lutomirski Cc: Borislav Petkov Cc: Dave Hansen Cc: Fenghua Yu Cc: H. Peter Anvin Cc: Linus Torvalds Cc: Oleg Nesterov Cc: Peter Zijlstra Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- arch/x86/include/asm/fpu/internal.h | 5 ++--- arch/x86/kernel/fpu/core.c | 6 ++---- arch/x86/kernel/process.c | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) (limited to 'arch/x86/kernel/process.c') diff --git a/arch/x86/include/asm/fpu/internal.h b/arch/x86/include/asm/fpu/internal.h index f20a0030f6a1..d6ac4611f05e 100644 --- a/arch/x86/include/asm/fpu/internal.h +++ b/arch/x86/include/asm/fpu/internal.h @@ -37,9 +37,8 @@ int ia32_setup_frame(int sig, struct ksignal *ksig, #define MXCSR_DEFAULT 0x1f80 extern unsigned int mxcsr_feature_mask; -extern void fpu__init_cpu(void); -extern void eager_fpu_init(void); +extern void fpu__init_cpu(void); extern void fpu__init_system_xstate(void); extern void fpu__init_cpu_xstate(void); extern void fpu__init_system(struct cpuinfo_x86 *c); @@ -57,7 +56,7 @@ extern void fpu__restore(void); extern void fpu__drop(struct fpu *fpu); extern int fpu__copy(struct fpu *dst_fpu, struct fpu *src_fpu); extern void fpu__reset(struct fpu *fpu); -extern void fpu__clear(struct task_struct *tsk); +extern void fpu__clear(struct fpu *fpu); extern void fpu__init_check_bugs(void); extern void fpu__resume_cpu(void); diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index acca83be23f0..0ccdd8348872 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -390,11 +390,9 @@ void fpu__reset(struct fpu *fpu) * Called by sys_execve() to clear the FPU fpregs, so that FPU state * of the previous binary does not leak over into the exec()ed binary: */ -void fpu__clear(struct task_struct *tsk) +void fpu__clear(struct fpu *fpu) { - struct fpu *fpu = &tsk->thread.fpu; - - WARN_ON_ONCE(tsk != current); /* Almost certainly an anomaly */ + WARN_ON_ONCE(fpu != ¤t->thread.fpu); /* Almost certainly an anomaly */ if (!use_eager_fpu()) { /* FPU state will be reallocated lazily at the first use. */ diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index dde263fb2031..b53f6c939bc2 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -120,7 +120,7 @@ void flush_thread(void) flush_ptrace_hw_breakpoint(tsk); memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array)); - fpu__clear(tsk); + fpu__clear(&tsk->thread.fpu); } static void hard_disable_TSC(void) -- cgit v1.2.3