From f3612304ee04a1a36ded7604771ea56d818158cb Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 17 Feb 2012 10:29:20 +0100 Subject: [S390] idle: avoid RCU usage in extended quiescent state Avoid calling wake_up() from our NMI "bottom halve" from RCU extended quiescent state in idle. wake_up() has RCU read-side critical sections but this will be completely ignored by RCU if the cpu is in extended quiescent state. Which means that whatever object is being accessed from within the read-side critical section can be freed concurrently from a different cpu. So make sure we leave extended quiescent state before calling wake_up(). Signed-off-by: Heiko Carstens Signed-off-by: Martin Schwidefsky --- arch/s390/kernel/process.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'arch/s390/kernel/process.c') diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 3201ae447990..4261aa799774 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -76,7 +76,6 @@ static void default_idle(void) if (test_thread_flag(TIF_MCCK_PENDING)) { local_mcck_enable(); local_irq_enable(); - s390_handle_mcck(); return; } trace_hardirqs_on(); @@ -93,10 +92,12 @@ void cpu_idle(void) for (;;) { tick_nohz_idle_enter(); rcu_idle_enter(); - while (!need_resched()) + while (!need_resched() && !test_thread_flag(TIF_MCCK_PENDING)) default_idle(); rcu_idle_exit(); tick_nohz_idle_exit(); + if (test_thread_flag(TIF_MCCK_PENDING)) + s390_handle_mcck(); preempt_enable_no_resched(); schedule(); preempt_disable(); -- cgit v1.2.3 From 048cd4e51d24ebf7f3552226d03c769d6ad91658 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 27 Feb 2012 10:01:52 +0100 Subject: compat: fix compile breakage on s390 The new is_compat_task() define for the !COMPAT case in include/linux/compat.h conflicts with a similar define in arch/s390/include/asm/compat.h. This is the minimal patch which fixes the build issues. Signed-off-by: Heiko Carstens Signed-off-by: Linus Torvalds --- arch/s390/include/asm/compat.h | 7 ------- arch/s390/kernel/process.c | 1 - arch/s390/kernel/ptrace.c | 2 +- arch/s390/kernel/setup.c | 2 +- arch/s390/kernel/signal.c | 1 - arch/s390/mm/fault.c | 1 - arch/s390/mm/mmap.c | 2 +- drivers/s390/block/dasd_eckd.c | 2 +- drivers/s390/block/dasd_ioctl.c | 1 + drivers/s390/char/fs3270.c | 1 + drivers/s390/char/vmcp.c | 1 + drivers/s390/cio/chsc_sch.c | 1 + drivers/s390/scsi/zfcp_cfdc.c | 1 + 13 files changed, 9 insertions(+), 14 deletions(-) (limited to 'arch/s390/kernel/process.c') diff --git a/arch/s390/include/asm/compat.h b/arch/s390/include/asm/compat.h index 2e49748b27da..234f1d859cea 100644 --- a/arch/s390/include/asm/compat.h +++ b/arch/s390/include/asm/compat.h @@ -172,13 +172,6 @@ static inline int is_compat_task(void) return is_32bit_task(); } -#else - -static inline int is_compat_task(void) -{ - return 0; -} - #endif static inline void __user *arch_compat_alloc_user_space(long len) diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 4261aa799774..e795933eb2cb 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -29,7 +29,6 @@ #include #include #include -#include #include #include "entry.h" diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 9d82ed4bcb27..61f95489d70c 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -20,8 +20,8 @@ #include #include #include +#include #include -#include #include #include #include diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 354de0763eff..3b2efc81f34e 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -46,6 +46,7 @@ #include #include #include +#include #include #include @@ -59,7 +60,6 @@ #include #include #include -#include #include #include diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c index a8ba840294ff..2d421d90fada 100644 --- a/arch/s390/kernel/signal.c +++ b/arch/s390/kernel/signal.c @@ -30,7 +30,6 @@ #include #include #include -#include #include "entry.h" #define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP))) diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 354dd39073ef..e8fcd928dc78 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -36,7 +36,6 @@ #include #include #include -#include #include "../kernel/entry.h" #ifndef CONFIG_64BIT diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index f09c74881b7e..a0155c02e324 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -29,8 +29,8 @@ #include #include #include +#include #include -#include static unsigned long stack_maxrandom_size(void) { diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 70880be26015..2617b1ed4709 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -18,12 +18,12 @@ #include /* HDIO_GETGEO */ #include #include +#include #include #include #include #include -#include #include #include #include diff --git a/drivers/s390/block/dasd_ioctl.c b/drivers/s390/block/dasd_ioctl.c index f1a2016829fc..792c69e78fe2 100644 --- a/drivers/s390/block/dasd_ioctl.c +++ b/drivers/s390/block/dasd_ioctl.c @@ -13,6 +13,7 @@ #define KMSG_COMPONENT "dasd" #include +#include #include #include #include diff --git a/drivers/s390/char/fs3270.c b/drivers/s390/char/fs3270.c index e71298158f9e..911704571b9c 100644 --- a/drivers/s390/char/fs3270.c +++ b/drivers/s390/char/fs3270.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/drivers/s390/char/vmcp.c b/drivers/s390/char/vmcp.c index 75bde6a8b7dc..89c03e6b1c0c 100644 --- a/drivers/s390/char/vmcp.c +++ b/drivers/s390/char/vmcp.c @@ -13,6 +13,7 @@ #include #include +#include #include #include #include diff --git a/drivers/s390/cio/chsc_sch.c b/drivers/s390/cio/chsc_sch.c index 0c87b0fc7714..8f9a1a384496 100644 --- a/drivers/s390/cio/chsc_sch.c +++ b/drivers/s390/cio/chsc_sch.c @@ -8,6 +8,7 @@ */ #include +#include #include #include #include diff --git a/drivers/s390/scsi/zfcp_cfdc.c b/drivers/s390/scsi/zfcp_cfdc.c index 303dde09d294..fab2c2592a97 100644 --- a/drivers/s390/scsi/zfcp_cfdc.c +++ b/drivers/s390/scsi/zfcp_cfdc.c @@ -11,6 +11,7 @@ #define KMSG_COMPONENT "zfcp" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#include #include #include #include -- cgit v1.2.3 From bd2f55361f18347e890d52ff9cfd8895455ec11b Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 21 Mar 2011 12:33:18 +0100 Subject: sched/rt: Use schedule_preempt_disabled() Coccinelle based conversion. Signed-off-by: Thomas Gleixner Acked-by: Peter Zijlstra Link: http://lkml.kernel.org/n/tip-24swm5zut3h9c4a6s46x8rws@git.kernel.org Signed-off-by: Ingo Molnar --- arch/arm/kernel/process.c | 4 +--- arch/avr32/kernel/process.c | 4 +--- arch/blackfin/kernel/process.c | 4 +--- arch/cris/kernel/process.c | 4 +--- arch/frv/kernel/process.c | 4 +--- arch/h8300/kernel/process.c | 4 +--- arch/ia64/kernel/process.c | 4 +--- arch/m32r/kernel/process.c | 4 +--- arch/m68k/kernel/process_mm.c | 4 +--- arch/m68k/kernel/process_no.c | 4 +--- arch/microblaze/kernel/process.c | 4 +--- arch/mips/kernel/process.c | 4 +--- arch/mn10300/kernel/process.c | 4 +--- arch/parisc/kernel/process.c | 4 +--- arch/powerpc/kernel/idle.c | 8 ++++---- arch/powerpc/platforms/iseries/setup.c | 8 ++------ arch/s390/kernel/process.c | 4 +--- arch/score/kernel/process.c | 4 +--- arch/sh/kernel/idle.c | 4 +--- arch/sparc/kernel/process_32.c | 8 ++------ arch/sparc/kernel/process_64.c | 10 ++++------ arch/tile/kernel/process.c | 4 +--- arch/x86/kernel/process_32.c | 4 +--- arch/x86/kernel/process_64.c | 4 +--- arch/xtensa/kernel/process.c | 4 +--- init/main.c | 5 +---- kernel/mutex.c | 4 +--- kernel/softirq.c | 4 +--- 28 files changed, 36 insertions(+), 95 deletions(-) (limited to 'arch/s390/kernel/process.c') diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 971d65c253a9..c2ae3cd331fe 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -239,9 +239,7 @@ void cpu_idle(void) leds_event(led_idle_end); rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/avr32/kernel/process.c b/arch/avr32/kernel/process.c index ea3395750324..92c5af98a6f7 100644 --- a/arch/avr32/kernel/process.c +++ b/arch/avr32/kernel/process.c @@ -40,9 +40,7 @@ void cpu_idle(void) cpu_idle_sleep(); rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/blackfin/kernel/process.c b/arch/blackfin/kernel/process.c index 8dd0416673cb..a80a643f3691 100644 --- a/arch/blackfin/kernel/process.c +++ b/arch/blackfin/kernel/process.c @@ -94,9 +94,7 @@ void cpu_idle(void) idle(); rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/cris/kernel/process.c b/arch/cris/kernel/process.c index aa585e4e979e..d8f50ff6fadd 100644 --- a/arch/cris/kernel/process.c +++ b/arch/cris/kernel/process.c @@ -115,9 +115,7 @@ void cpu_idle (void) idle = default_idle; idle(); } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/frv/kernel/process.c b/arch/frv/kernel/process.c index 3901df1213c0..29cc49783787 100644 --- a/arch/frv/kernel/process.c +++ b/arch/frv/kernel/process.c @@ -92,9 +92,7 @@ void cpu_idle(void) idle(); } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/h8300/kernel/process.c b/arch/h8300/kernel/process.c index 933bd388efb2..1a173b35f475 100644 --- a/arch/h8300/kernel/process.c +++ b/arch/h8300/kernel/process.c @@ -81,9 +81,7 @@ void cpu_idle(void) while (1) { while (!need_resched()) idle(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/ia64/kernel/process.c b/arch/ia64/kernel/process.c index 6d33c5cc94f0..9dc52b63fc87 100644 --- a/arch/ia64/kernel/process.c +++ b/arch/ia64/kernel/process.c @@ -330,9 +330,7 @@ cpu_idle (void) normal_xtp(); #endif } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); check_pgt_cache(); if (cpu_is_offline(cpu)) play_dead(); diff --git a/arch/m32r/kernel/process.c b/arch/m32r/kernel/process.c index 422bea9f1dbc..3a4a32b27208 100644 --- a/arch/m32r/kernel/process.c +++ b/arch/m32r/kernel/process.c @@ -90,9 +90,7 @@ void cpu_idle (void) idle(); } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/m68k/kernel/process_mm.c b/arch/m68k/kernel/process_mm.c index 099283ee1a8f..fe4186b5fc32 100644 --- a/arch/m68k/kernel/process_mm.c +++ b/arch/m68k/kernel/process_mm.c @@ -78,9 +78,7 @@ void cpu_idle(void) while (1) { while (!need_resched()) idle(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/m68k/kernel/process_no.c b/arch/m68k/kernel/process_no.c index 5e1078cabe0e..f7fe6c348595 100644 --- a/arch/m68k/kernel/process_no.c +++ b/arch/m68k/kernel/process_no.c @@ -73,9 +73,7 @@ void cpu_idle(void) /* endless idle loop with no priority at all */ while (1) { idle(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/microblaze/kernel/process.c b/arch/microblaze/kernel/process.c index 7dcb5bfffb75..9155f7d92669 100644 --- a/arch/microblaze/kernel/process.c +++ b/arch/microblaze/kernel/process.c @@ -110,9 +110,7 @@ void cpu_idle(void) rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); check_pgt_cache(); } } diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index 7955409051c4..61f1cb45a1d5 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -80,9 +80,7 @@ void __noreturn cpu_idle(void) #endif rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/mn10300/kernel/process.c b/arch/mn10300/kernel/process.c index 28eec3102535..cac401d37f75 100644 --- a/arch/mn10300/kernel/process.c +++ b/arch/mn10300/kernel/process.c @@ -123,9 +123,7 @@ void cpu_idle(void) idle(); } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/parisc/kernel/process.c b/arch/parisc/kernel/process.c index 62c60b87d039..d4b94b395c16 100644 --- a/arch/parisc/kernel/process.c +++ b/arch/parisc/kernel/process.c @@ -71,9 +71,7 @@ void cpu_idle(void) while (1) { while (!need_resched()) barrier(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); check_pgt_cache(); } } diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c index 0a48bf5db6c8..65035141552b 100644 --- a/arch/powerpc/kernel/idle.c +++ b/arch/powerpc/kernel/idle.c @@ -101,11 +101,11 @@ void cpu_idle(void) ppc64_runlatch_on(); rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - if (cpu_should_die()) + if (cpu_should_die()) { + preempt_enable_no_resched(); cpu_die(); - schedule(); - preempt_disable(); + } + schedule_preempt_disabled(); } } diff --git a/arch/powerpc/platforms/iseries/setup.c b/arch/powerpc/platforms/iseries/setup.c index 8fc62586a973..a5fbf4cb6329 100644 --- a/arch/powerpc/platforms/iseries/setup.c +++ b/arch/powerpc/platforms/iseries/setup.c @@ -584,9 +584,7 @@ static void iseries_shared_idle(void) if (hvlpevent_is_pending()) process_iSeries_events(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } @@ -615,9 +613,7 @@ static void iseries_dedicated_idle(void) ppc64_runlatch_on(); rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index e795933eb2cb..7618085b4164 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -97,9 +97,7 @@ void cpu_idle(void) tick_nohz_idle_exit(); if (test_thread_flag(TIF_MCCK_PENDING)) s390_handle_mcck(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/score/kernel/process.c b/arch/score/kernel/process.c index 25d08030a883..2707023c7563 100644 --- a/arch/score/kernel/process.c +++ b/arch/score/kernel/process.c @@ -53,9 +53,7 @@ void __noreturn cpu_idle(void) while (!need_resched()) barrier(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/sh/kernel/idle.c b/arch/sh/kernel/idle.c index 406508d4ce74..7e4892826563 100644 --- a/arch/sh/kernel/idle.c +++ b/arch/sh/kernel/idle.c @@ -114,9 +114,7 @@ void cpu_idle(void) rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/sparc/kernel/process_32.c b/arch/sparc/kernel/process_32.c index f793742eec2b..935fdbcd88c2 100644 --- a/arch/sparc/kernel/process_32.c +++ b/arch/sparc/kernel/process_32.c @@ -113,9 +113,7 @@ void cpu_idle(void) while (!need_resched()) cpu_relax(); } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); check_pgt_cache(); } } @@ -138,9 +136,7 @@ void cpu_idle(void) while (!need_resched()) cpu_relax(); } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); check_pgt_cache(); } } diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index 39d8b05201a2..ab9a29268213 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -104,15 +104,13 @@ void cpu_idle(void) rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - #ifdef CONFIG_HOTPLUG_CPU - if (cpu_is_offline(cpu)) + if (cpu_is_offline(cpu)) { + preempt_enable_no_resched(); cpu_play_dead(); + } #endif - - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/tile/kernel/process.c b/arch/tile/kernel/process.c index 4c1ac6e5347a..6ae495ef2b99 100644 --- a/arch/tile/kernel/process.c +++ b/arch/tile/kernel/process.c @@ -108,9 +108,7 @@ void cpu_idle(void) } rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c index c08d1ff12b7c..49888fefe794 100644 --- a/arch/x86/kernel/process_32.c +++ b/arch/x86/kernel/process_32.c @@ -119,9 +119,7 @@ void cpu_idle(void) } rcu_idle_exit(); tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index cfa5c90c01db..e34257c70c28 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -156,9 +156,7 @@ void cpu_idle(void) } tick_nohz_idle_exit(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/arch/xtensa/kernel/process.c b/arch/xtensa/kernel/process.c index 47041e7c088c..2c9004770c4e 100644 --- a/arch/xtensa/kernel/process.c +++ b/arch/xtensa/kernel/process.c @@ -113,9 +113,7 @@ void cpu_idle(void) while (1) { while (!need_resched()) platform_idle(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } } diff --git a/init/main.c b/init/main.c index ff49a6dacfbb..4990f7ec776a 100644 --- a/init/main.c +++ b/init/main.c @@ -374,11 +374,8 @@ static noinline void __init_refok rest_init(void) * at least once to get things moving: */ init_idle_bootup_task(current); - preempt_enable_no_resched(); - schedule(); - + schedule_preempt_disabled(); /* Call into cpu_idle with preempt disabled */ - preempt_disable(); cpu_idle(); } diff --git a/kernel/mutex.c b/kernel/mutex.c index 89096dd8786f..a307cc9c9526 100644 --- a/kernel/mutex.c +++ b/kernel/mutex.c @@ -240,9 +240,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, /* didn't get the lock, go to sleep: */ spin_unlock_mutex(&lock->wait_lock, flags); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); spin_lock_mutex(&lock->wait_lock, flags); } diff --git a/kernel/softirq.c b/kernel/softirq.c index 4eb3a0fa351e..79b524767a24 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -744,9 +744,7 @@ static int run_ksoftirqd(void * __bind_cpu) while (!kthread_should_stop()) { preempt_disable(); if (!local_softirq_pending()) { - preempt_enable_no_resched(); - schedule(); - preempt_disable(); + schedule_preempt_disabled(); } __set_current_state(TASK_RUNNING); -- cgit v1.2.3 From 4c1051e37a0e2a941115c6fb7ba08c318f25a0f9 Mon Sep 17 00:00:00 2001 From: Martin Schwidefsky Date: Sun, 11 Mar 2012 11:59:27 -0400 Subject: [S390] rework idle code Whenever the cpu loads an enabled wait PSW it will appear as idle to the underlying host system. The code in default_idle calls vtime_stop_cpu which does the necessary voodoo to get the cpu time accounting right. The udelay code just loads an enabled wait PSW. To correct this rework the vtime_stop_cpu/vtime_start_cpu logic and move the difficult parts to entry[64].S, vtime_stop_cpu can now be called from anywhere and vtime_start_cpu is gone. The correction of the cpu time during wakeup from an enabled wait PSW is done with a critical section in entry[64].S. As vtime_start_cpu is gone, s390_idle_check can be removed as well. Signed-off-by: Martin Schwidefsky --- arch/s390/include/asm/cputime.h | 9 +-- arch/s390/include/asm/timer.h | 4 +- arch/s390/kernel/asm-offsets.c | 8 ++ arch/s390/kernel/entry.S | 76 +++++++++++++++++-- arch/s390/kernel/entry.h | 5 +- arch/s390/kernel/entry64.S | 65 +++++++++++++++- arch/s390/kernel/irq.c | 2 - arch/s390/kernel/nmi.c | 2 - arch/s390/kernel/process.c | 7 +- arch/s390/kernel/smp.c | 46 +++++------- arch/s390/kernel/vtime.c | 163 ++++++++-------------------------------- arch/s390/lib/delay.c | 31 ++++---- drivers/s390/cio/cio.c | 2 - 13 files changed, 210 insertions(+), 210 deletions(-) (limited to 'arch/s390/kernel/process.c') diff --git a/arch/s390/include/asm/cputime.h b/arch/s390/include/asm/cputime.h index c23c3900c304..24ef186a1c4f 100644 --- a/arch/s390/include/asm/cputime.h +++ b/arch/s390/include/asm/cputime.h @@ -170,24 +170,17 @@ struct s390_idle_data { unsigned int sequence; unsigned long long idle_count; unsigned long long idle_enter; + unsigned long long idle_exit; unsigned long long idle_time; int nohz_delay; }; DECLARE_PER_CPU(struct s390_idle_data, s390_idle); -void vtime_start_cpu(__u64 int_clock, __u64 enter_timer); cputime64_t s390_get_idle_time(int cpu); #define arch_idle_time(cpu) s390_get_idle_time(cpu) -static inline void s390_idle_check(struct pt_regs *regs, __u64 int_clock, - __u64 enter_timer) -{ - if (regs->psw.mask & PSW_MASK_WAIT) - vtime_start_cpu(int_clock, enter_timer); -} - static inline int s390_nohz_delay(int cpu) { return __get_cpu_var(s390_idle).nohz_delay != 0; diff --git a/arch/s390/include/asm/timer.h b/arch/s390/include/asm/timer.h index 814243cafdfe..e63069ba39e3 100644 --- a/arch/s390/include/asm/timer.h +++ b/arch/s390/include/asm/timer.h @@ -33,8 +33,8 @@ struct vtimer_queue { spinlock_t lock; __u64 timer; /* last programmed timer */ __u64 elapsed; /* elapsed time of timer expire values */ - __u64 idle; /* temp var for idle */ - int do_spt; /* =1: reprogram cpu timer in idle */ + __u64 idle_enter; /* cpu timer on idle enter */ + __u64 idle_exit; /* cpu timer on idle exit */ }; extern void init_virt_timer(struct vtimer_list *timer); diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index aeeaf896be9b..ed8c913db79e 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -8,6 +8,8 @@ #include #include +#include +#include #include #include #include @@ -70,6 +72,12 @@ int main(void) DEFINE(__CLOCK_MONOTONIC, CLOCK_MONOTONIC); DEFINE(__CLOCK_REALTIME_RES, MONOTONIC_RES_NSEC); BLANK(); + /* idle data offsets */ + DEFINE(__IDLE_ENTER, offsetof(struct s390_idle_data, idle_enter)); + DEFINE(__IDLE_EXIT, offsetof(struct s390_idle_data, idle_exit)); + /* vtimer queue offsets */ + DEFINE(__VQ_IDLE_ENTER, offsetof(struct vtimer_queue, idle_enter)); + DEFINE(__VQ_IDLE_EXIT, offsetof(struct vtimer_queue, idle_exit)); /* lowcore offsets */ DEFINE(__LC_EXT_PARAMS, offsetof(struct _lowcore, ext_params)); DEFINE(__LC_EXT_CPU_ADDR, offsetof(struct _lowcore, ext_cpu_addr)); diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 6143521a4fff..74ee563fe62b 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -105,14 +105,14 @@ STACK_SIZE = 1 << STACK_SHIFT .macro ADD64 high,low,timer al \high,\timer - al \low,\timer+4 + al \low,4+\timer brc 12,.+8 ahi \high,1 .endm .macro SUB64 high,low,timer sl \high,\timer - sl \low,\timer+4 + sl \low,4+\timer brc 3,.+8 ahi \high,-1 .endm @@ -471,7 +471,6 @@ io_tif: jnz io_work # there is work to do (signals etc.) io_restore: mvc __LC_RETURN_PSW(8),__PT_PSW(%r11) - ni __LC_RETURN_PSW+1,0xfd # clean wait state bit stpt __LC_EXIT_TIMER lm %r0,%r15,__PT_R0(%r11) lpsw __LC_RETURN_PSW @@ -612,6 +611,26 @@ ext_skip: basr %r14,%r1 # call do_extint j io_return +/* + * Load idle PSW. The second "half" of this function is in cleanup_idle. + */ +ENTRY(psw_idle) + st %r4,__SF_EMPTY(%r15) + basr %r1,0 + la %r1,psw_idle_lpsw+4-.(%r1) + st %r1,__SF_EMPTY+4(%r15) + oi __SF_EMPTY+4(%r15),0x80 + la %r1,.Lvtimer_max-psw_idle_lpsw-4(%r1) + stck __IDLE_ENTER(%r2) + ltr %r5,%r5 + stpt __VQ_IDLE_ENTER(%r3) + jz psw_idle_lpsw + spt 0(%r1) +psw_idle_lpsw: + lpsw __SF_EMPTY(%r15) + br %r14 +psw_idle_end: + __critical_end: /* @@ -673,7 +692,6 @@ mcck_skip: TRACE_IRQS_ON mcck_return: mvc __LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW - ni __LC_RETURN_MCCK_PSW+1,0xfd # clear wait state bit tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? jno 0f lm %r0,%r15,__PT_R0(%r11) @@ -748,6 +766,8 @@ cleanup_table: .long io_tif + 0x80000000 .long io_restore + 0x80000000 .long io_done + 0x80000000 + .long psw_idle + 0x80000000 + .long psw_idle_end + 0x80000000 cleanup_critical: cl %r9,BASED(cleanup_table) # system_call @@ -766,6 +786,10 @@ cleanup_critical: jl cleanup_io_tif cl %r9,BASED(cleanup_table+28) # io_done jl cleanup_io_restore + cl %r9,BASED(cleanup_table+32) # psw_idle + jl 0f + cl %r9,BASED(cleanup_table+36) # psw_idle_end + jl cleanup_idle 0: br %r14 cleanup_system_call: @@ -849,7 +873,6 @@ cleanup_io_restore: jhe 0f l %r9,12(%r11) # get saved r11 pointer to pt_regs mvc __LC_RETURN_PSW(8),__PT_PSW(%r9) - ni __LC_RETURN_PSW+1,0xfd # clear wait state bit mvc 0(32,%r11),__PT_R8(%r9) lm %r0,%r7,__PT_R0(%r9) 0: lm %r8,%r9,__LC_RETURN_PSW @@ -857,11 +880,52 @@ cleanup_io_restore: cleanup_io_restore_insn: .long io_done - 4 + 0x80000000 +cleanup_idle: + # copy interrupt clock & cpu timer + mvc __IDLE_EXIT(8,%r2),__LC_INT_CLOCK + mvc __VQ_IDLE_EXIT(8,%r3),__LC_ASYNC_ENTER_TIMER + chi %r11,__LC_SAVE_AREA_ASYNC + je 0f + mvc __IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK + mvc __VQ_IDLE_EXIT(8,%r3),__LC_MCCK_ENTER_TIMER +0: # check if stck has been executed + cl %r9,BASED(cleanup_idle_insn) + jhe 1f + mvc __IDLE_ENTER(8,%r2),__IDLE_EXIT(%r2) + mvc __VQ_IDLE_ENTER(8,%r3),__VQ_IDLE_EXIT(%r3) + j 2f +1: # check if the cpu timer has been reprogrammed + ltr %r5,%r5 + jz 2f + spt __VQ_IDLE_ENTER(%r3) +2: # account system time going idle + lm %r9,%r10,__LC_STEAL_TIMER + ADD64 %r9,%r10,__IDLE_ENTER(%r2) + SUB64 %r9,%r10,__LC_LAST_UPDATE_CLOCK + stm %r9,%r10,__LC_STEAL_TIMER + mvc __LC_LAST_UPDATE_CLOCK(8),__IDLE_EXIT(%r2) + lm %r9,%r10,__LC_SYSTEM_TIMER + ADD64 %r9,%r10,__LC_LAST_UPDATE_TIMER + SUB64 %r9,%r10,__VQ_IDLE_ENTER(%r3) + stm %r9,%r10,__LC_SYSTEM_TIMER + mvc __LC_LAST_UPDATE_TIMER(8),__VQ_IDLE_EXIT(%r3) + # prepare return psw + n %r8,BASED(cleanup_idle_wait) # clear wait state bit + l %r9,24(%r11) # return from psw_idle + br %r14 +cleanup_idle_insn: + .long psw_idle_lpsw + 0x80000000 +cleanup_idle_wait: + .long 0xfffdffff + /* * Integer constants */ .align 4 -.Lnr_syscalls: .long NR_syscalls +.Lnr_syscalls: + .long NR_syscalls +.Lvtimer_max: + .quad 0x7fffffffffffffff /* * Symbol constants diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index 92b1617d0c95..4984785e3078 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -4,7 +4,8 @@ #include #include #include - +#include +#include extern void (*pgm_check_table[128])(struct pt_regs *); extern void *restart_stack; @@ -16,6 +17,8 @@ void io_int_handler(void); void mcck_int_handler(void); void restart_int_handler(void); void restart_call_handler(void); +void psw_idle(struct s390_idle_data *, struct vtimer_queue *, + unsigned long, int); asmlinkage long do_syscall_trace_enter(struct pt_regs *regs); asmlinkage void do_syscall_trace_exit(struct pt_regs *regs); diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S index e33789a45752..4e1c292fa7e3 100644 --- a/arch/s390/kernel/entry64.S +++ b/arch/s390/kernel/entry64.S @@ -489,7 +489,6 @@ io_restore: lg %r14,__LC_VDSO_PER_CPU lmg %r0,%r10,__PT_R0(%r11) mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) - ni __LC_RETURN_PSW+1,0xfd # clear wait state bit stpt __LC_EXIT_TIMER mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER lmg %r11,%r15,__PT_R11(%r11) @@ -631,6 +630,24 @@ ext_skip: brasl %r14,do_extint j io_return +/* + * Load idle PSW. The second "half" of this function is in cleanup_idle. + */ +ENTRY(psw_idle) + stg %r4,__SF_EMPTY(%r15) + larl %r1,psw_idle_lpsw+4 + stg %r1,__SF_EMPTY+8(%r15) + larl %r1,.Lvtimer_max + stck __IDLE_ENTER(%r2) + ltr %r5,%r5 + stpt __VQ_IDLE_ENTER(%r3) + jz psw_idle_lpsw + spt 0(%r1) +psw_idle_lpsw: + lpswe __SF_EMPTY(%r15) + br %r14 +psw_idle_end: + __critical_end: /* @@ -696,7 +713,6 @@ mcck_return: lg %r14,__LC_VDSO_PER_CPU lmg %r0,%r10,__PT_R0(%r11) mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW - ni __LC_RETURN_MCCK_PSW+1,0xfd # clear wait state bit tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? jno 0f stpt __LC_EXIT_TIMER @@ -770,6 +786,8 @@ cleanup_table: .quad io_tif .quad io_restore .quad io_done + .quad psw_idle + .quad psw_idle_end cleanup_critical: clg %r9,BASED(cleanup_table) # system_call @@ -788,6 +806,10 @@ cleanup_critical: jl cleanup_io_tif clg %r9,BASED(cleanup_table+56) # io_done jl cleanup_io_restore + clg %r9,BASED(cleanup_table+64) # psw_idle + jl 0f + clg %r9,BASED(cleanup_table+72) # psw_idle_end + jl cleanup_idle 0: br %r14 @@ -877,7 +899,6 @@ cleanup_io_restore: je 0f lg %r9,24(%r11) # get saved r11 pointer to pt_regs mvc __LC_RETURN_PSW(16),__PT_PSW(%r9) - ni __LC_RETURN_PSW+1,0xfd # clear wait state bit mvc 0(64,%r11),__PT_R8(%r9) lmg %r0,%r7,__PT_R0(%r9) 0: lmg %r8,%r9,__LC_RETURN_PSW @@ -885,6 +906,42 @@ cleanup_io_restore: cleanup_io_restore_insn: .quad io_done - 4 +cleanup_idle: + # copy interrupt clock & cpu timer + mvc __IDLE_EXIT(8,%r2),__LC_INT_CLOCK + mvc __VQ_IDLE_EXIT(8,%r3),__LC_ASYNC_ENTER_TIMER + cghi %r11,__LC_SAVE_AREA_ASYNC + je 0f + mvc __IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK + mvc __VQ_IDLE_EXIT(8,%r3),__LC_MCCK_ENTER_TIMER +0: # check if stck & stpt have been executed + clg %r9,BASED(cleanup_idle_insn) + jhe 1f + mvc __IDLE_ENTER(8,%r2),__IDLE_EXIT(%r2) + mvc __VQ_IDLE_ENTER(8,%r3),__VQ_IDLE_EXIT(%r3) + j 2f +1: # check if the cpu timer has been reprogrammed + ltr %r5,%r5 + jz 2f + spt __VQ_IDLE_ENTER(%r3) +2: # account system time going idle + lg %r9,__LC_STEAL_TIMER + alg %r9,__IDLE_ENTER(%r2) + slg %r9,__LC_LAST_UPDATE_CLOCK + stg %r9,__LC_STEAL_TIMER + mvc __LC_LAST_UPDATE_CLOCK(8),__IDLE_EXIT(%r2) + lg %r9,__LC_SYSTEM_TIMER + alg %r9,__LC_LAST_UPDATE_TIMER + slg %r9,__VQ_IDLE_ENTER(%r3) + stg %r9,__LC_SYSTEM_TIMER + mvc __LC_LAST_UPDATE_TIMER(8),__VQ_IDLE_EXIT(%r3) + # prepare return psw + nihh %r8,0xfffd # clear wait state bit + lg %r9,48(%r11) # return from psw_idle + br %r14 +cleanup_idle_insn: + .quad psw_idle_lpsw + /* * Integer constants */ @@ -893,6 +950,8 @@ cleanup_io_restore_insn: .quad __critical_start .Lcritical_length: .quad __critical_end - __critical_start +.Lvtimer_max: + .quad 0x7fffffffffffffff #if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE) diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index b9a7fdd9c814..09a014c62537 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -219,8 +219,6 @@ void __irq_entry do_extint(struct pt_regs *regs, unsigned int ext_int_code, code = (unsigned short) ext_int_code; old_regs = set_irq_regs(regs); - s390_idle_check(regs, S390_lowcore.int_clock, - S390_lowcore.async_enter_timer); irq_enter(); if (S390_lowcore.int_clock >= S390_lowcore.clock_comparator) /* Serve timer interrupts first. */ diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 0fd2e863e114..8c372ca61350 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -254,8 +254,6 @@ void notrace s390_do_machine_check(struct pt_regs *regs) int umode; nmi_enter(); - s390_idle_check(regs, S390_lowcore.mcck_clock, - S390_lowcore.mcck_enter_timer); kstat_cpu(smp_processor_id()).irqs[NMI_NMI]++; mci = (struct mci *) &S390_lowcore.mcck_interruption_code; mcck = &__get_cpu_var(cpu_mcck); diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index e795933eb2cb..78b3c149b8b6 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -77,13 +77,8 @@ static void default_idle(void) local_irq_enable(); return; } - trace_hardirqs_on(); - /* Don't trace preempt off for idle. */ - stop_critical_timings(); - /* Stop virtual timer and halt the cpu. */ + /* Halt the cpu and keep track of cpu time accounting. */ vtime_stop_cpu(); - /* Reenable preemption tracer. */ - start_critical_timings(); } void cpu_idle(void) diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 6db8526a602d..afd6e5113a90 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -972,22 +972,16 @@ static DEVICE_ATTR(capability, 0444, show_capability, NULL); static ssize_t show_idle_count(struct device *dev, struct device_attribute *attr, char *buf) { - struct s390_idle_data *idle; + struct s390_idle_data *idle = &per_cpu(s390_idle, dev->id); unsigned long long idle_count; unsigned int sequence; - idle = &per_cpu(s390_idle, dev->id); -repeat: - sequence = idle->sequence; - smp_rmb(); - if (sequence & 1) - goto repeat; - idle_count = idle->idle_count; - if (idle->idle_enter) - idle_count++; - smp_rmb(); - if (idle->sequence != sequence) - goto repeat; + do { + sequence = ACCESS_ONCE(idle->sequence); + idle_count = ACCESS_ONCE(idle->idle_count); + if (ACCESS_ONCE(idle->idle_enter)) + idle_count++; + } while ((sequence & 1) || (idle->sequence != sequence)); return sprintf(buf, "%llu\n", idle_count); } static DEVICE_ATTR(idle_count, 0444, show_idle_count, NULL); @@ -995,24 +989,18 @@ static DEVICE_ATTR(idle_count, 0444, show_idle_count, NULL); static ssize_t show_idle_time(struct device *dev, struct device_attribute *attr, char *buf) { - struct s390_idle_data *idle; - unsigned long long now, idle_time, idle_enter; + struct s390_idle_data *idle = &per_cpu(s390_idle, dev->id); + unsigned long long now, idle_time, idle_enter, idle_exit; unsigned int sequence; - idle = &per_cpu(s390_idle, dev->id); - now = get_clock(); -repeat: - sequence = idle->sequence; - smp_rmb(); - if (sequence & 1) - goto repeat; - idle_time = idle->idle_time; - idle_enter = idle->idle_enter; - if (idle_enter != 0ULL && idle_enter < now) - idle_time += now - idle_enter; - smp_rmb(); - if (idle->sequence != sequence) - goto repeat; + do { + now = get_clock(); + sequence = ACCESS_ONCE(idle->sequence); + idle_time = ACCESS_ONCE(idle->idle_time); + idle_enter = ACCESS_ONCE(idle->idle_enter); + idle_exit = ACCESS_ONCE(idle->idle_exit); + } while ((sequence & 1) || (idle->sequence != sequence)); + idle_time += idle_enter ? ((idle_exit ? : now) - idle_enter) : 0; return sprintf(buf, "%llu\n", idle_time >> 12); } static DEVICE_ATTR(idle_time_us, 0444, show_idle_time, NULL); diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 7bacee9a546f..277ea712b232 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -26,6 +26,7 @@ #include #include #include +#include "entry.h" static DEFINE_PER_CPU(struct vtimer_queue, virt_cpu_timer); @@ -123,153 +124,53 @@ void account_system_vtime(struct task_struct *tsk) } EXPORT_SYMBOL_GPL(account_system_vtime); -void __kprobes vtime_start_cpu(__u64 int_clock, __u64 enter_timer) +void __kprobes vtime_stop_cpu(void) { struct s390_idle_data *idle = &__get_cpu_var(s390_idle); struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer); - __u64 idle_time, expires; + unsigned long long idle_time; + unsigned long psw_mask; - if (idle->idle_enter == 0ULL) - return; + trace_hardirqs_on(); + /* Don't trace preempt off for idle. */ + stop_critical_timings(); - /* Account time spent with enabled wait psw loaded as idle time. */ - idle_time = int_clock - idle->idle_enter; - account_idle_time(idle_time); - S390_lowcore.steal_timer += - idle->idle_enter - S390_lowcore.last_update_clock; - S390_lowcore.last_update_clock = int_clock; - - /* Account system time spent going idle. */ - S390_lowcore.system_timer += S390_lowcore.last_update_timer - vq->idle; - S390_lowcore.last_update_timer = enter_timer; - - /* Restart vtime CPU timer */ - if (vq->do_spt) { - /* Program old expire value but first save progress. */ - expires = vq->idle - enter_timer; - expires += get_vtimer(); - set_vtimer(expires); - } else { - /* Don't account the CPU timer delta while the cpu was idle. */ - vq->elapsed -= vq->idle - enter_timer; - } + /* Wait for external, I/O or machine check interrupt. */ + psw_mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_DAT | + PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK; + idle->nohz_delay = 0; + /* Call the assembler magic in entry.S */ + psw_idle(idle, vq, psw_mask, !list_empty(&vq->list)); + + /* Reenable preemption tracer. */ + start_critical_timings(); + + /* Account time spent with enabled wait psw loaded as idle time. */ idle->sequence++; smp_wmb(); + idle_time = idle->idle_exit - idle->idle_enter; idle->idle_time += idle_time; - idle->idle_enter = 0ULL; + idle->idle_enter = idle->idle_exit = 0ULL; idle->idle_count++; + account_idle_time(idle_time); smp_wmb(); idle->sequence++; } -void __kprobes vtime_stop_cpu(void) -{ - struct s390_idle_data *idle = &__get_cpu_var(s390_idle); - struct vtimer_queue *vq = &__get_cpu_var(virt_cpu_timer); - psw_t psw; - - /* Wait for external, I/O or machine check interrupt. */ - psw.mask = psw_kernel_bits | PSW_MASK_WAIT | - PSW_MASK_DAT | PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK; - - idle->nohz_delay = 0; - - /* Check if the CPU timer needs to be reprogrammed. */ - if (vq->do_spt) { - __u64 vmax = VTIMER_MAX_SLICE; - /* - * The inline assembly is equivalent to - * vq->idle = get_cpu_timer(); - * set_cpu_timer(VTIMER_MAX_SLICE); - * idle->idle_enter = get_clock(); - * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT | - * PSW_MASK_DAT | PSW_MASK_IO | - * PSW_MASK_EXT | PSW_MASK_MCHECK); - * The difference is that the inline assembly makes sure that - * the last three instruction are stpt, stck and lpsw in that - * order. This is done to increase the precision. - */ - asm volatile( -#ifndef CONFIG_64BIT - " basr 1,0\n" - "0: ahi 1,1f-0b\n" - " st 1,4(%2)\n" -#else /* CONFIG_64BIT */ - " larl 1,1f\n" - " stg 1,8(%2)\n" -#endif /* CONFIG_64BIT */ - " stpt 0(%4)\n" - " spt 0(%5)\n" - " stck 0(%3)\n" -#ifndef CONFIG_64BIT - " lpsw 0(%2)\n" -#else /* CONFIG_64BIT */ - " lpswe 0(%2)\n" -#endif /* CONFIG_64BIT */ - "1:" - : "=m" (idle->idle_enter), "=m" (vq->idle) - : "a" (&psw), "a" (&idle->idle_enter), - "a" (&vq->idle), "a" (&vmax), "m" (vmax), "m" (psw) - : "memory", "cc", "1"); - } else { - /* - * The inline assembly is equivalent to - * vq->idle = get_cpu_timer(); - * idle->idle_enter = get_clock(); - * __load_psw_mask(psw_kernel_bits | PSW_MASK_WAIT | - * PSW_MASK_DAT | PSW_MASK_IO | - * PSW_MASK_EXT | PSW_MASK_MCHECK); - * The difference is that the inline assembly makes sure that - * the last three instruction are stpt, stck and lpsw in that - * order. This is done to increase the precision. - */ - asm volatile( -#ifndef CONFIG_64BIT - " basr 1,0\n" - "0: ahi 1,1f-0b\n" - " st 1,4(%2)\n" -#else /* CONFIG_64BIT */ - " larl 1,1f\n" - " stg 1,8(%2)\n" -#endif /* CONFIG_64BIT */ - " stpt 0(%4)\n" - " stck 0(%3)\n" -#ifndef CONFIG_64BIT - " lpsw 0(%2)\n" -#else /* CONFIG_64BIT */ - " lpswe 0(%2)\n" -#endif /* CONFIG_64BIT */ - "1:" - : "=m" (idle->idle_enter), "=m" (vq->idle) - : "a" (&psw), "a" (&idle->idle_enter), - "a" (&vq->idle), "m" (psw) - : "memory", "cc", "1"); - } -} - cputime64_t s390_get_idle_time(int cpu) { - struct s390_idle_data *idle; - unsigned long long now, idle_time, idle_enter; + struct s390_idle_data *idle = &per_cpu(s390_idle, cpu); + unsigned long long now, idle_enter, idle_exit; unsigned int sequence; - idle = &per_cpu(s390_idle, cpu); - - now = get_clock(); -repeat: - sequence = idle->sequence; - smp_rmb(); - if (sequence & 1) - goto repeat; - idle_time = 0; - idle_enter = idle->idle_enter; - if (idle_enter != 0ULL && idle_enter < now) - idle_time = now - idle_enter; - smp_rmb(); - if (idle->sequence != sequence) - goto repeat; - return idle_time; + do { + now = get_clock(); + sequence = ACCESS_ONCE(idle->sequence); + idle_enter = ACCESS_ONCE(idle->idle_enter); + idle_exit = ACCESS_ONCE(idle->idle_exit); + } while ((sequence & 1) || (idle->sequence != sequence)); + return idle_enter ? ((idle_exit ? : now) - idle_enter) : 0; } /* @@ -346,7 +247,6 @@ static void do_cpu_timer_interrupt(unsigned int ext_int_code, } spin_unlock(&vq->lock); - vq->do_spt = list_empty(&cb_list); do_callbacks(&cb_list); /* next event is first in list */ @@ -355,8 +255,7 @@ static void do_cpu_timer_interrupt(unsigned int ext_int_code, if (!list_empty(&vq->list)) { event = list_first_entry(&vq->list, struct vtimer_list, entry); next = event->expires; - } else - vq->do_spt = 0; + } spin_unlock(&vq->lock); /* * To improve precision add the time spent by the diff --git a/arch/s390/lib/delay.c b/arch/s390/lib/delay.c index db92f044024c..9f1f71e85778 100644 --- a/arch/s390/lib/delay.c +++ b/arch/s390/lib/delay.c @@ -13,6 +13,7 @@ #include #include #include +#include void __delay(unsigned long loops) { @@ -28,36 +29,33 @@ void __delay(unsigned long loops) static void __udelay_disabled(unsigned long long usecs) { - unsigned long mask, cr0, cr0_saved; - u64 clock_saved; - u64 end; + unsigned long cr0, cr6, new; + u64 clock_saved, end; - mask = psw_kernel_bits | PSW_MASK_DAT | PSW_MASK_WAIT | - PSW_MASK_EXT | PSW_MASK_MCHECK; end = get_clock() + (usecs << 12); clock_saved = local_tick_disable(); - __ctl_store(cr0_saved, 0, 0); - cr0 = (cr0_saved & 0xffff00e0) | 0x00000800; - __ctl_load(cr0 , 0, 0); + __ctl_store(cr0, 0, 0); + __ctl_store(cr6, 6, 6); + new = (cr0 & 0xffff00e0) | 0x00000800; + __ctl_load(new , 0, 0); + new = 0; + __ctl_load(new, 6, 6); lockdep_off(); do { set_clock_comparator(end); - trace_hardirqs_on(); - __load_psw_mask(mask); + vtime_stop_cpu(); local_irq_disable(); } while (get_clock() < end); lockdep_on(); - __ctl_load(cr0_saved, 0, 0); + __ctl_load(cr0, 0, 0); + __ctl_load(cr6, 6, 6); local_tick_enable(clock_saved); } static void __udelay_enabled(unsigned long long usecs) { - unsigned long mask; - u64 clock_saved; - u64 end; + u64 clock_saved, end; - mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_EXT | PSW_MASK_IO; end = get_clock() + (usecs << 12); do { clock_saved = 0; @@ -65,8 +63,7 @@ static void __udelay_enabled(unsigned long long usecs) clock_saved = local_tick_disable(); set_clock_comparator(end); } - trace_hardirqs_on(); - __load_psw_mask(mask); + vtime_stop_cpu(); local_irq_disable(); if (clock_saved) local_tick_enable(clock_saved); diff --git a/drivers/s390/cio/cio.c b/drivers/s390/cio/cio.c index dc67c397449e..a49c46c91983 100644 --- a/drivers/s390/cio/cio.c +++ b/drivers/s390/cio/cio.c @@ -601,8 +601,6 @@ void __irq_entry do_IRQ(struct pt_regs *regs) struct pt_regs *old_regs; old_regs = set_irq_regs(regs); - s390_idle_check(regs, S390_lowcore.int_clock, - S390_lowcore.async_enter_timer); irq_enter(); __this_cpu_write(s390_idle.nohz_delay, 1); if (S390_lowcore.int_clock >= S390_lowcore.clock_comparator) -- cgit v1.2.3 From a0616cdebcfd575dcd4c46102d1b52fbb827fc29 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 28 Mar 2012 18:30:02 +0100 Subject: Disintegrate asm/system.h for S390 Disintegrate asm/system.h for S390. Signed-off-by: David Howells cc: linux-s390@vger.kernel.org --- arch/s390/crypto/crypt_s390.h | 1 + arch/s390/include/asm/atomic.h | 2 +- arch/s390/include/asm/barrier.h | 35 ++++ arch/s390/include/asm/ctl_reg.h | 76 +++++++++ arch/s390/include/asm/elf.h | 1 - arch/s390/include/asm/exec.h | 12 ++ arch/s390/include/asm/facility.h | 63 +++++++ arch/s390/include/asm/mmu.h | 14 ++ arch/s390/include/asm/mmu_context.h | 1 + arch/s390/include/asm/processor.h | 24 +++ arch/s390/include/asm/setup.h | 14 ++ arch/s390/include/asm/smp.h | 2 +- arch/s390/include/asm/switch_to.h | 100 +++++++++++ arch/s390/include/asm/system.h | 320 +----------------------------------- arch/s390/include/asm/uaccess.h | 6 + arch/s390/kernel/asm-offsets.c | 1 - arch/s390/kernel/compat_signal.c | 1 + arch/s390/kernel/cpcmd.c | 1 - arch/s390/kernel/dis.c | 1 - arch/s390/kernel/early.c | 2 +- arch/s390/kernel/lgr.c | 1 - arch/s390/kernel/machine_kexec.c | 1 - arch/s390/kernel/os_info.c | 1 - arch/s390/kernel/process.c | 2 +- arch/s390/kernel/ptrace.c | 2 +- arch/s390/kernel/setup.c | 1 - arch/s390/kernel/signal.c | 1 + arch/s390/kernel/suspend.c | 2 +- arch/s390/kernel/traps.c | 1 - arch/s390/kernel/vdso.c | 2 +- arch/s390/kvm/kvm-s390.c | 2 +- arch/s390/mm/fault.c | 2 +- arch/s390/mm/init.c | 2 +- arch/s390/mm/maccess.c | 2 +- arch/s390/mm/pgtable.c | 1 - arch/s390/oprofile/hwsampler.c | 1 + drivers/s390/char/sclp_cmd.c | 1 + drivers/s390/cio/crw.c | 1 + drivers/s390/crypto/ap_bus.c | 1 + 39 files changed, 369 insertions(+), 335 deletions(-) create mode 100644 arch/s390/include/asm/barrier.h create mode 100644 arch/s390/include/asm/ctl_reg.h create mode 100644 arch/s390/include/asm/exec.h create mode 100644 arch/s390/include/asm/facility.h create mode 100644 arch/s390/include/asm/switch_to.h (limited to 'arch/s390/kernel/process.c') diff --git a/arch/s390/crypto/crypt_s390.h b/arch/s390/crypto/crypt_s390.h index ffd1ac255f19..9178db6db0a5 100644 --- a/arch/s390/crypto/crypt_s390.h +++ b/arch/s390/crypto/crypt_s390.h @@ -17,6 +17,7 @@ #define _CRYPTO_ARCH_S390_CRYPT_S390_H #include +#include #define CRYPT_S390_OP_MASK 0xFF00 #define CRYPT_S390_FUNC_MASK 0x00FF diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index 8517d2ae3b5c..748347baecb8 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -15,7 +15,7 @@ #include #include -#include +#include #define ATOMIC_INIT(i) { (i) } diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h new file mode 100644 index 000000000000..451273ad4d34 --- /dev/null +++ b/arch/s390/include/asm/barrier.h @@ -0,0 +1,35 @@ +/* + * Copyright IBM Corp. 1999, 2009 + * + * Author(s): Martin Schwidefsky + */ + +#ifndef __ASM_BARRIER_H +#define __ASM_BARRIER_H + +/* + * Force strict CPU ordering. + * And yes, this is required on UP too when we're talking + * to devices. + * + * This is very similar to the ppc eieio/sync instruction in that is + * does a checkpoint syncronisation & makes sure that + * all memory ops have completed wrt other CPU's ( see 7-15 POP DJB ). + */ + +#define eieio() asm volatile("bcr 15,0" : : : "memory") +#define SYNC_OTHER_CORES(x) eieio() +#define mb() eieio() +#define rmb() eieio() +#define wmb() eieio() +#define read_barrier_depends() do { } while(0) +#define smp_mb() mb() +#define smp_rmb() rmb() +#define smp_wmb() wmb() +#define smp_read_barrier_depends() read_barrier_depends() +#define smp_mb__before_clear_bit() smp_mb() +#define smp_mb__after_clear_bit() smp_mb() + +#define set_mb(var, value) do { var = value; mb(); } while (0) + +#endif /* __ASM_BARRIER_H */ diff --git a/arch/s390/include/asm/ctl_reg.h b/arch/s390/include/asm/ctl_reg.h new file mode 100644 index 000000000000..ecde9417d669 --- /dev/null +++ b/arch/s390/include/asm/ctl_reg.h @@ -0,0 +1,76 @@ +/* + * Copyright IBM Corp. 1999, 2009 + * + * Author(s): Martin Schwidefsky + */ + +#ifndef __ASM_CTL_REG_H +#define __ASM_CTL_REG_H + +#ifdef __s390x__ + +#define __ctl_load(array, low, high) ({ \ + typedef struct { char _[sizeof(array)]; } addrtype; \ + asm volatile( \ + " lctlg %1,%2,%0\n" \ + : : "Q" (*(addrtype *)(&array)), \ + "i" (low), "i" (high)); \ + }) + +#define __ctl_store(array, low, high) ({ \ + typedef struct { char _[sizeof(array)]; } addrtype; \ + asm volatile( \ + " stctg %1,%2,%0\n" \ + : "=Q" (*(addrtype *)(&array)) \ + : "i" (low), "i" (high)); \ + }) + +#else /* __s390x__ */ + +#define __ctl_load(array, low, high) ({ \ + typedef struct { char _[sizeof(array)]; } addrtype; \ + asm volatile( \ + " lctl %1,%2,%0\n" \ + : : "Q" (*(addrtype *)(&array)), \ + "i" (low), "i" (high)); \ +}) + +#define __ctl_store(array, low, high) ({ \ + typedef struct { char _[sizeof(array)]; } addrtype; \ + asm volatile( \ + " stctl %1,%2,%0\n" \ + : "=Q" (*(addrtype *)(&array)) \ + : "i" (low), "i" (high)); \ + }) + +#endif /* __s390x__ */ + +#define __ctl_set_bit(cr, bit) ({ \ + unsigned long __dummy; \ + __ctl_store(__dummy, cr, cr); \ + __dummy |= 1UL << (bit); \ + __ctl_load(__dummy, cr, cr); \ +}) + +#define __ctl_clear_bit(cr, bit) ({ \ + unsigned long __dummy; \ + __ctl_store(__dummy, cr, cr); \ + __dummy &= ~(1UL << (bit)); \ + __ctl_load(__dummy, cr, cr); \ +}) + +#ifdef CONFIG_SMP + +extern void smp_ctl_set_bit(int cr, int bit); +extern void smp_ctl_clear_bit(int cr, int bit); +#define ctl_set_bit(cr, bit) smp_ctl_set_bit(cr, bit) +#define ctl_clear_bit(cr, bit) smp_ctl_clear_bit(cr, bit) + +#else + +#define ctl_set_bit(cr, bit) __ctl_set_bit(cr, bit) +#define ctl_clear_bit(cr, bit) __ctl_clear_bit(cr, bit) + +#endif /* CONFIG_SMP */ + +#endif /* __ASM_CTL_REG_H */ diff --git a/arch/s390/include/asm/elf.h b/arch/s390/include/asm/elf.h index 547f1a6a35d4..c4ee39f7a4d6 100644 --- a/arch/s390/include/asm/elf.h +++ b/arch/s390/include/asm/elf.h @@ -129,7 +129,6 @@ typedef s390_fp_regs compat_elf_fpregset_t; typedef s390_compat_regs compat_elf_gregset_t; #include /* for task_struct */ -#include /* for save_access_regs */ #include #include diff --git a/arch/s390/include/asm/exec.h b/arch/s390/include/asm/exec.h new file mode 100644 index 000000000000..c4a93d6327fa --- /dev/null +++ b/arch/s390/include/asm/exec.h @@ -0,0 +1,12 @@ +/* + * Copyright IBM Corp. 1999, 2009 + * + * Author(s): Martin Schwidefsky + */ + +#ifndef __ASM_EXEC_H +#define __ASM_EXEC_H + +extern unsigned long arch_align_stack(unsigned long sp); + +#endif /* __ASM_EXEC_H */ diff --git a/arch/s390/include/asm/facility.h b/arch/s390/include/asm/facility.h new file mode 100644 index 000000000000..1e5b27edc0c9 --- /dev/null +++ b/arch/s390/include/asm/facility.h @@ -0,0 +1,63 @@ +/* + * Copyright IBM Corp. 1999, 2009 + * + * Author(s): Martin Schwidefsky + */ + +#ifndef __ASM_FACILITY_H +#define __ASM_FACILITY_H + +#include +#include +#include + +#define MAX_FACILITY_BIT (256*8) /* stfle_fac_list has 256 bytes */ + +/* + * The test_facility function uses the bit odering where the MSB is bit 0. + * That makes it easier to query facility bits with the bit number as + * documented in the Principles of Operation. + */ +static inline int test_facility(unsigned long nr) +{ + unsigned char *ptr; + + if (nr >= MAX_FACILITY_BIT) + return 0; + ptr = (unsigned char *) &S390_lowcore.stfle_fac_list + (nr >> 3); + return (*ptr & (0x80 >> (nr & 7))) != 0; +} + +/** + * stfle - Store facility list extended + * @stfle_fac_list: array where facility list can be stored + * @size: size of passed in array in double words + */ +static inline void stfle(u64 *stfle_fac_list, int size) +{ + unsigned long nr; + + preempt_disable(); + S390_lowcore.stfl_fac_list = 0; + asm volatile( + " .insn s,0xb2b10000,0(0)\n" /* stfl */ + "0:\n" + EX_TABLE(0b, 0b) + : "=m" (S390_lowcore.stfl_fac_list)); + nr = 4; /* bytes stored by stfl */ + memcpy(stfle_fac_list, &S390_lowcore.stfl_fac_list, 4); + if (S390_lowcore.stfl_fac_list & 0x01000000) { + /* More facility bits available with stfle */ + register unsigned long reg0 asm("0") = size - 1; + + asm volatile(".insn s,0xb2b00000,0(%1)" /* stfle */ + : "+d" (reg0) + : "a" (stfle_fac_list) + : "memory", "cc"); + nr = (reg0 + 1) * 8; /* # bytes stored by stfle */ + } + memset((char *) stfle_fac_list + nr, 0, size * 8 - nr); + preempt_enable(); +} + +#endif /* __ASM_FACILITY_H */ diff --git a/arch/s390/include/asm/mmu.h b/arch/s390/include/asm/mmu.h index 4506791adcd5..1c7d6ce328bf 100644 --- a/arch/s390/include/asm/mmu.h +++ b/arch/s390/include/asm/mmu.h @@ -21,4 +21,18 @@ typedef struct { .context.pgtable_list = LIST_HEAD_INIT(name.context.pgtable_list), \ .context.gmap_list = LIST_HEAD_INIT(name.context.gmap_list), +static inline int tprot(unsigned long addr) +{ + int rc = -EFAULT; + + asm volatile( + " tprot 0(%1),0\n" + "0: ipm %0\n" + " srl %0,28\n" + "1:\n" + EX_TABLE(0b,1b) + : "+d" (rc) : "a" (addr) : "cc"); + return rc; +} + #endif diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index 5682f160ff82..5d09e405c54d 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -12,6 +12,7 @@ #include #include #include +#include #include static inline int init_new_context(struct task_struct *tsk, diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index d25843a6a915..d499b30ea487 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -14,6 +14,7 @@ #define __ASM_S390_PROCESSOR_H #include +#include #include #include #include @@ -156,6 +157,14 @@ unsigned long get_wchan(struct task_struct *p); #define KSTK_EIP(tsk) (task_pt_regs(tsk)->psw.addr) #define KSTK_ESP(tsk) (task_pt_regs(tsk)->gprs[15]) +static inline unsigned short stap(void) +{ + unsigned short cpu_address; + + asm volatile("stap %0" : "=m" (cpu_address)); + return cpu_address; +} + /* * Give up the time slice of the virtual PU. */ @@ -303,6 +312,21 @@ static inline void __noreturn disabled_wait(unsigned long code) while (1); } +/* + * Use to set psw mask except for the first byte which + * won't be changed by this function. + */ +static inline void +__set_psw_mask(unsigned long mask) +{ + __load_psw_mask(mask | (arch_local_save_flags() & ~(-1UL >> 8))); +} + +#define local_mcck_enable() \ + __set_psw_mask(psw_kernel_bits | PSW_MASK_DAT | PSW_MASK_MCHECK) +#define local_mcck_disable() \ + __set_psw_mask(psw_kernel_bits | PSW_MASK_DAT) + /* * Basic Machine Check/Program Check Handler. */ diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index 097183c70407..b21e46e5d4b8 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -140,6 +140,20 @@ extern char vmpoff_cmd[]; #define NSS_NAME_SIZE 8 extern char kernel_nss_name[]; +#ifdef CONFIG_PFAULT +extern int pfault_init(void); +extern void pfault_fini(void); +#else /* CONFIG_PFAULT */ +#define pfault_init() ({-1;}) +#define pfault_fini() do { } while (0) +#endif /* CONFIG_PFAULT */ + +extern void cmma_init(void); + +extern void (*_machine_restart)(char *command); +extern void (*_machine_halt)(void); +extern void (*_machine_power_off)(void); + #else /* __ASSEMBLY__ */ #ifndef __s390x__ diff --git a/arch/s390/include/asm/smp.h b/arch/s390/include/asm/smp.h index 797f78729680..c77c6de6f6c0 100644 --- a/arch/s390/include/asm/smp.h +++ b/arch/s390/include/asm/smp.h @@ -9,7 +9,7 @@ #ifdef CONFIG_SMP -#include +#include #define raw_smp_processor_id() (S390_lowcore.cpu_nr) diff --git a/arch/s390/include/asm/switch_to.h b/arch/s390/include/asm/switch_to.h new file mode 100644 index 000000000000..f223068b7822 --- /dev/null +++ b/arch/s390/include/asm/switch_to.h @@ -0,0 +1,100 @@ +/* + * Copyright IBM Corp. 1999, 2009 + * + * Author(s): Martin Schwidefsky + */ + +#ifndef __ASM_SWITCH_TO_H +#define __ASM_SWITCH_TO_H + +#include + +extern struct task_struct *__switch_to(void *, void *); +extern void update_per_regs(struct task_struct *task); + +static inline void save_fp_regs(s390_fp_regs *fpregs) +{ + asm volatile( + " std 0,%O0+8(%R0)\n" + " std 2,%O0+24(%R0)\n" + " std 4,%O0+40(%R0)\n" + " std 6,%O0+56(%R0)" + : "=Q" (*fpregs) : "Q" (*fpregs)); + if (!MACHINE_HAS_IEEE) + return; + asm volatile( + " stfpc %0\n" + " std 1,%O0+16(%R0)\n" + " std 3,%O0+32(%R0)\n" + " std 5,%O0+48(%R0)\n" + " std 7,%O0+64(%R0)\n" + " std 8,%O0+72(%R0)\n" + " std 9,%O0+80(%R0)\n" + " std 10,%O0+88(%R0)\n" + " std 11,%O0+96(%R0)\n" + " std 12,%O0+104(%R0)\n" + " std 13,%O0+112(%R0)\n" + " std 14,%O0+120(%R0)\n" + " std 15,%O0+128(%R0)\n" + : "=Q" (*fpregs) : "Q" (*fpregs)); +} + +static inline void restore_fp_regs(s390_fp_regs *fpregs) +{ + asm volatile( + " ld 0,%O0+8(%R0)\n" + " ld 2,%O0+24(%R0)\n" + " ld 4,%O0+40(%R0)\n" + " ld 6,%O0+56(%R0)" + : : "Q" (*fpregs)); + if (!MACHINE_HAS_IEEE) + return; + asm volatile( + " lfpc %0\n" + " ld 1,%O0+16(%R0)\n" + " ld 3,%O0+32(%R0)\n" + " ld 5,%O0+48(%R0)\n" + " ld 7,%O0+64(%R0)\n" + " ld 8,%O0+72(%R0)\n" + " ld 9,%O0+80(%R0)\n" + " ld 10,%O0+88(%R0)\n" + " ld 11,%O0+96(%R0)\n" + " ld 12,%O0+104(%R0)\n" + " ld 13,%O0+112(%R0)\n" + " ld 14,%O0+120(%R0)\n" + " ld 15,%O0+128(%R0)\n" + : : "Q" (*fpregs)); +} + +static inline void save_access_regs(unsigned int *acrs) +{ + asm volatile("stam 0,15,%0" : "=Q" (*acrs)); +} + +static inline void restore_access_regs(unsigned int *acrs) +{ + asm volatile("lam 0,15,%0" : : "Q" (*acrs)); +} + +#define switch_to(prev,next,last) do { \ + if (prev->mm) { \ + save_fp_regs(&prev->thread.fp_regs); \ + save_access_regs(&prev->thread.acrs[0]); \ + } \ + if (next->mm) { \ + restore_fp_regs(&next->thread.fp_regs); \ + restore_access_regs(&next->thread.acrs[0]); \ + update_per_regs(next); \ + } \ + prev = __switch_to(prev,next); \ +} while (0) + +extern void account_vtime(struct task_struct *, struct task_struct *); +extern void account_tick_vtime(struct task_struct *); + +#define finish_arch_switch(prev) do { \ + set_fs(current->thread.mm_segment); \ + account_vtime(prev, current); \ +} while (0) + +#endif /* __ASM_SWITCH_TO_H */ diff --git a/arch/s390/include/asm/system.h b/arch/s390/include/asm/system.h index 2e0bb7f0f9b2..641c72903277 100644 --- a/arch/s390/include/asm/system.h +++ b/arch/s390/include/asm/system.h @@ -1,315 +1,7 @@ -/* - * Copyright IBM Corp. 1999, 2009 - * - * Author(s): Martin Schwidefsky - */ - -#ifndef __ASM_SYSTEM_H -#define __ASM_SYSTEM_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include +/* FILE TO BE DELETED. DO NOT ADD STUFF HERE! */ +#include #include - -#ifdef __KERNEL__ - -struct task_struct; - -extern struct task_struct *__switch_to(void *, void *); -extern void update_per_regs(struct task_struct *task); - -static inline void save_fp_regs(s390_fp_regs *fpregs) -{ - asm volatile( - " std 0,%O0+8(%R0)\n" - " std 2,%O0+24(%R0)\n" - " std 4,%O0+40(%R0)\n" - " std 6,%O0+56(%R0)" - : "=Q" (*fpregs) : "Q" (*fpregs)); - if (!MACHINE_HAS_IEEE) - return; - asm volatile( - " stfpc %0\n" - " std 1,%O0+16(%R0)\n" - " std 3,%O0+32(%R0)\n" - " std 5,%O0+48(%R0)\n" - " std 7,%O0+64(%R0)\n" - " std 8,%O0+72(%R0)\n" - " std 9,%O0+80(%R0)\n" - " std 10,%O0+88(%R0)\n" - " std 11,%O0+96(%R0)\n" - " std 12,%O0+104(%R0)\n" - " std 13,%O0+112(%R0)\n" - " std 14,%O0+120(%R0)\n" - " std 15,%O0+128(%R0)\n" - : "=Q" (*fpregs) : "Q" (*fpregs)); -} - -static inline void restore_fp_regs(s390_fp_regs *fpregs) -{ - asm volatile( - " ld 0,%O0+8(%R0)\n" - " ld 2,%O0+24(%R0)\n" - " ld 4,%O0+40(%R0)\n" - " ld 6,%O0+56(%R0)" - : : "Q" (*fpregs)); - if (!MACHINE_HAS_IEEE) - return; - asm volatile( - " lfpc %0\n" - " ld 1,%O0+16(%R0)\n" - " ld 3,%O0+32(%R0)\n" - " ld 5,%O0+48(%R0)\n" - " ld 7,%O0+64(%R0)\n" - " ld 8,%O0+72(%R0)\n" - " ld 9,%O0+80(%R0)\n" - " ld 10,%O0+88(%R0)\n" - " ld 11,%O0+96(%R0)\n" - " ld 12,%O0+104(%R0)\n" - " ld 13,%O0+112(%R0)\n" - " ld 14,%O0+120(%R0)\n" - " ld 15,%O0+128(%R0)\n" - : : "Q" (*fpregs)); -} - -static inline void save_access_regs(unsigned int *acrs) -{ - asm volatile("stam 0,15,%0" : "=Q" (*acrs)); -} - -static inline void restore_access_regs(unsigned int *acrs) -{ - asm volatile("lam 0,15,%0" : : "Q" (*acrs)); -} - -#define switch_to(prev,next,last) do { \ - if (prev->mm) { \ - save_fp_regs(&prev->thread.fp_regs); \ - save_access_regs(&prev->thread.acrs[0]); \ - } \ - if (next->mm) { \ - restore_fp_regs(&next->thread.fp_regs); \ - restore_access_regs(&next->thread.acrs[0]); \ - update_per_regs(next); \ - } \ - prev = __switch_to(prev,next); \ -} while (0) - -extern void account_vtime(struct task_struct *, struct task_struct *); -extern void account_tick_vtime(struct task_struct *); - -#ifdef CONFIG_PFAULT -extern int pfault_init(void); -extern void pfault_fini(void); -#else /* CONFIG_PFAULT */ -#define pfault_init() ({-1;}) -#define pfault_fini() do { } while (0) -#endif /* CONFIG_PFAULT */ - -extern void cmma_init(void); -extern int memcpy_real(void *, void *, size_t); -extern void copy_to_absolute_zero(void *dest, void *src, size_t count); -extern int copy_to_user_real(void __user *dest, void *src, size_t count); -extern int copy_from_user_real(void *dest, void __user *src, size_t count); - -#define finish_arch_switch(prev) do { \ - set_fs(current->thread.mm_segment); \ - account_vtime(prev, current); \ -} while (0) - -#define nop() asm volatile("nop") - -/* - * Force strict CPU ordering. - * And yes, this is required on UP too when we're talking - * to devices. - * - * This is very similar to the ppc eieio/sync instruction in that is - * does a checkpoint syncronisation & makes sure that - * all memory ops have completed wrt other CPU's ( see 7-15 POP DJB ). - */ - -#define eieio() asm volatile("bcr 15,0" : : : "memory") -#define SYNC_OTHER_CORES(x) eieio() -#define mb() eieio() -#define rmb() eieio() -#define wmb() eieio() -#define read_barrier_depends() do { } while(0) -#define smp_mb() mb() -#define smp_rmb() rmb() -#define smp_wmb() wmb() -#define smp_read_barrier_depends() read_barrier_depends() -#define smp_mb__before_clear_bit() smp_mb() -#define smp_mb__after_clear_bit() smp_mb() - - -#define set_mb(var, value) do { var = value; mb(); } while (0) - -#ifdef __s390x__ - -#define __ctl_load(array, low, high) ({ \ - typedef struct { char _[sizeof(array)]; } addrtype; \ - asm volatile( \ - " lctlg %1,%2,%0\n" \ - : : "Q" (*(addrtype *)(&array)), \ - "i" (low), "i" (high)); \ - }) - -#define __ctl_store(array, low, high) ({ \ - typedef struct { char _[sizeof(array)]; } addrtype; \ - asm volatile( \ - " stctg %1,%2,%0\n" \ - : "=Q" (*(addrtype *)(&array)) \ - : "i" (low), "i" (high)); \ - }) - -#else /* __s390x__ */ - -#define __ctl_load(array, low, high) ({ \ - typedef struct { char _[sizeof(array)]; } addrtype; \ - asm volatile( \ - " lctl %1,%2,%0\n" \ - : : "Q" (*(addrtype *)(&array)), \ - "i" (low), "i" (high)); \ -}) - -#define __ctl_store(array, low, high) ({ \ - typedef struct { char _[sizeof(array)]; } addrtype; \ - asm volatile( \ - " stctl %1,%2,%0\n" \ - : "=Q" (*(addrtype *)(&array)) \ - : "i" (low), "i" (high)); \ - }) - -#endif /* __s390x__ */ - -#define __ctl_set_bit(cr, bit) ({ \ - unsigned long __dummy; \ - __ctl_store(__dummy, cr, cr); \ - __dummy |= 1UL << (bit); \ - __ctl_load(__dummy, cr, cr); \ -}) - -#define __ctl_clear_bit(cr, bit) ({ \ - unsigned long __dummy; \ - __ctl_store(__dummy, cr, cr); \ - __dummy &= ~(1UL << (bit)); \ - __ctl_load(__dummy, cr, cr); \ -}) - -/* - * Use to set psw mask except for the first byte which - * won't be changed by this function. - */ -static inline void -__set_psw_mask(unsigned long mask) -{ - __load_psw_mask(mask | (arch_local_save_flags() & ~(-1UL >> 8))); -} - -#define local_mcck_enable() \ - __set_psw_mask(psw_kernel_bits | PSW_MASK_DAT | PSW_MASK_MCHECK) -#define local_mcck_disable() \ - __set_psw_mask(psw_kernel_bits | PSW_MASK_DAT) - -#ifdef CONFIG_SMP - -extern void smp_ctl_set_bit(int cr, int bit); -extern void smp_ctl_clear_bit(int cr, int bit); -#define ctl_set_bit(cr, bit) smp_ctl_set_bit(cr, bit) -#define ctl_clear_bit(cr, bit) smp_ctl_clear_bit(cr, bit) - -#else - -#define ctl_set_bit(cr, bit) __ctl_set_bit(cr, bit) -#define ctl_clear_bit(cr, bit) __ctl_clear_bit(cr, bit) - -#endif /* CONFIG_SMP */ - -#define MAX_FACILITY_BIT (256*8) /* stfle_fac_list has 256 bytes */ - -/* - * The test_facility function uses the bit odering where the MSB is bit 0. - * That makes it easier to query facility bits with the bit number as - * documented in the Principles of Operation. - */ -static inline int test_facility(unsigned long nr) -{ - unsigned char *ptr; - - if (nr >= MAX_FACILITY_BIT) - return 0; - ptr = (unsigned char *) &S390_lowcore.stfle_fac_list + (nr >> 3); - return (*ptr & (0x80 >> (nr & 7))) != 0; -} - -/** - * stfle - Store facility list extended - * @stfle_fac_list: array where facility list can be stored - * @size: size of passed in array in double words - */ -static inline void stfle(u64 *stfle_fac_list, int size) -{ - unsigned long nr; - - preempt_disable(); - S390_lowcore.stfl_fac_list = 0; - asm volatile( - " .insn s,0xb2b10000,0(0)\n" /* stfl */ - "0:\n" - EX_TABLE(0b, 0b) - : "=m" (S390_lowcore.stfl_fac_list)); - nr = 4; /* bytes stored by stfl */ - memcpy(stfle_fac_list, &S390_lowcore.stfl_fac_list, 4); - if (S390_lowcore.stfl_fac_list & 0x01000000) { - /* More facility bits available with stfle */ - register unsigned long reg0 asm("0") = size - 1; - - asm volatile(".insn s,0xb2b00000,0(%1)" /* stfle */ - : "+d" (reg0) - : "a" (stfle_fac_list) - : "memory", "cc"); - nr = (reg0 + 1) * 8; /* # bytes stored by stfle */ - } - memset((char *) stfle_fac_list + nr, 0, size * 8 - nr); - preempt_enable(); -} - -static inline unsigned short stap(void) -{ - unsigned short cpu_address; - - asm volatile("stap %0" : "=m" (cpu_address)); - return cpu_address; -} - -extern void (*_machine_restart)(char *command); -extern void (*_machine_halt)(void); -extern void (*_machine_power_off)(void); - -extern unsigned long arch_align_stack(unsigned long sp); - -static inline int tprot(unsigned long addr) -{ - int rc = -EFAULT; - - asm volatile( - " tprot 0(%1),0\n" - "0: ipm %0\n" - " srl %0,28\n" - "1:\n" - EX_TABLE(0b,1b) - : "+d" (rc) : "a" (addr) : "cc"); - return rc; -} - -#endif /* __KERNEL__ */ - -#endif +#include +#include +#include +#include diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index 2b23885e81e9..8f2cada4f7c9 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -16,6 +16,7 @@ */ #include #include +#include #define VERIFY_READ 0 #define VERIFY_WRITE 1 @@ -375,4 +376,9 @@ clear_user(void __user *to, unsigned long n) return n; } +extern int memcpy_real(void *, void *, size_t); +extern void copy_to_absolute_zero(void *dest, void *src, size_t count); +extern int copy_to_user_real(void __user *dest, void *src, size_t count); +extern int copy_from_user_real(void *dest, void __user *src, size_t count); + #endif /* __S390_UACCESS_H */ diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index ed8c913db79e..83e6edf5cf17 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -12,7 +12,6 @@ #include #include #include -#include /* * Make sure that the compiler is new enough. We want a compiler that diff --git a/arch/s390/kernel/compat_signal.c b/arch/s390/kernel/compat_signal.c index 53a82c8d50e9..28040fd5e8a2 100644 --- a/arch/s390/kernel/compat_signal.c +++ b/arch/s390/kernel/compat_signal.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "compat_linux.h" #include "compat_ptrace.h" #include "entry.h" diff --git a/arch/s390/kernel/cpcmd.c b/arch/s390/kernel/cpcmd.c index 3e8b8816f309..e3dd886e1b32 100644 --- a/arch/s390/kernel/cpcmd.c +++ b/arch/s390/kernel/cpcmd.c @@ -18,7 +18,6 @@ #include #include #include -#include #include static DEFINE_SPINLOCK(cpcmd_lock); diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index e2f847599c8e..3221c6fca8bb 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -24,7 +24,6 @@ #include #include -#include #include #include #include diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 578eb4e6d157..9475e682727f 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include "entry.h" /* diff --git a/arch/s390/kernel/lgr.c b/arch/s390/kernel/lgr.c index 8431b92ca3ae..ac39e7a731fc 100644 --- a/arch/s390/kernel/lgr.c +++ b/arch/s390/kernel/lgr.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index 0f8cdf1268d0..bdad47d54478 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/s390/kernel/os_info.c b/arch/s390/kernel/os_info.c index bbe522672e06..e8d6c214d498 100644 --- a/arch/s390/kernel/os_info.c +++ b/arch/s390/kernel/os_info.c @@ -12,7 +12,6 @@ #include #include #include -#include #include /* diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 3732e4c09cbe..60055cefdd04 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -23,13 +23,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include #include "entry.h" asmlinkage void ret_from_fork(void) asm ("ret_from_fork"); diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index 61f95489d70c..02f300fbf070 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -26,9 +26,9 @@ #include #include #include -#include #include #include +#include #include "entry.h" #ifdef CONFIG_COMPAT diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 38e751278bf7..1581ea2e027a 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -50,7 +50,6 @@ #include #include -#include #include #include #include diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c index f29f5ef400e5..f7582b27f600 100644 --- a/arch/s390/kernel/signal.c +++ b/arch/s390/kernel/signal.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "entry.h" #define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP))) diff --git a/arch/s390/kernel/suspend.c b/arch/s390/kernel/suspend.c index 47df775c844d..aa1494d0e380 100644 --- a/arch/s390/kernel/suspend.c +++ b/arch/s390/kernel/suspend.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include /* * References to section boundaries diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index cd6ebe12c481..77cdf4234ebc 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index e704a9965f90..d95427e32a56 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -25,12 +25,12 @@ #include #include #include -#include #include #include #include #include #include +#include #if defined(CONFIG_32BIT) || defined(CONFIG_COMPAT) extern char vdso32_start, vdso32_end; diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index d1c445732451..d56de16a651a 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -27,7 +27,7 @@ #include #include #include -#include +#include #include "kvm-s390.h" #include "gaccess.h" diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index b17c42df61c9..46ef3fd0663b 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -32,10 +32,10 @@ #include #include #include -#include #include #include #include +#include #include "../kernel/entry.h" #ifndef CONFIG_64BIT diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 50236610de83..2bea0605856e 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include @@ -38,6 +37,7 @@ #include #include #include +#include pgd_t swapper_pg_dir[PTRS_PER_PGD] __attribute__((__aligned__(PAGE_SIZE))); diff --git a/arch/s390/mm/maccess.c b/arch/s390/mm/maccess.c index 1cb8427bedfb..7bb15fcca75e 100644 --- a/arch/s390/mm/maccess.c +++ b/arch/s390/mm/maccess.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include /* * This function writes to kernel memory bypassing DAT and possible diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index 51b0738e13d1..373adf69b01c 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -18,7 +18,6 @@ #include #include -#include #include #include #include diff --git a/arch/s390/oprofile/hwsampler.c b/arch/s390/oprofile/hwsampler.c index 12bea05a0fc1..59c9278befc9 100644 --- a/arch/s390/oprofile/hwsampler.c +++ b/arch/s390/oprofile/hwsampler.c @@ -18,6 +18,7 @@ #include #include +#include #include #include diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c index 0b54a91f8dcd..2e58f472f7f6 100644 --- a/drivers/s390/char/sclp_cmd.c +++ b/drivers/s390/char/sclp_cmd.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "sclp.h" diff --git a/drivers/s390/cio/crw.c b/drivers/s390/cio/crw.c index 425f741a280c..d0a2dff43fb4 100644 --- a/drivers/s390/cio/crw.c +++ b/drivers/s390/cio/crw.c @@ -13,6 +13,7 @@ #include #include #include +#include static DEFINE_MUTEX(crw_handler_mutex); static crw_handler_t crw_handlers[NR_RSCS]; diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 12ae1817b172..c5c121f14c99 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -46,6 +46,7 @@ #include #include #include +#include #include "ap_bus.h" -- cgit v1.2.3