diff options
author | Stephen Rothwell <sfr@canb.auug.org.au> | 2008-06-13 17:03:43 +1000 |
---|---|---|
committer | Stephen Rothwell <sfr@canb.auug.org.au> | 2008-06-13 17:09:14 +1000 |
commit | 2448ad3d8e64a9b58acca34a3d010cfa0be88cc1 (patch) | |
tree | efd58c12106c4b35fbd777255a5bfdf68430cfa4 /arch | |
parent | 11c04206578c83651c72f3703b0015e5bfa4ec84 (diff) | |
parent | e780915f6042ef29158b4d86e7028cffc3609f8e (diff) |
Merge commit 'block/for-next'
Conflicts:
arch/powerpc/Kconfig
arch/x86/kernel/apic_32.c
arch/x86/kernel/i8259_64.c
arch/x86/xen/smp.c
include/asm-x86/hw_irq_32.h
include/asm-x86/hw_irq_64.h
include/asm-x86/mach-default/irq_vectors.h
include/asm-x86/mach-voyager/irq_vectors.h
kernel/Makefile
Diffstat (limited to 'arch')
36 files changed, 245 insertions, 1518 deletions
diff --git a/arch/Kconfig b/arch/Kconfig index 3ea332b009e5..ad89a33d8c6e 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -39,3 +39,6 @@ config HAVE_KRETPROBES config HAVE_DMA_ATTRS def_bool n + +config USE_GENERIC_SMP_HELPERS + def_bool n diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig index 729cdbdf8036..dbe8c280fea9 100644 --- a/arch/alpha/Kconfig +++ b/arch/alpha/Kconfig @@ -528,6 +528,7 @@ config ARCH_MAY_HAVE_PC_FDC config SMP bool "Symmetric multi-processing support" depends on ALPHA_SABLE || ALPHA_LYNX || ALPHA_RAWHIDE || ALPHA_DP264 || ALPHA_WILDFIRE || ALPHA_TITAN || ALPHA_GENERIC || ALPHA_SHARK || ALPHA_MARVEL + select USE_GENERIC_SMP_HELPERS ---help--- This enables support for systems with more than one CPU. If you have a system with only one CPU, like most personal computers, say N. If diff --git a/arch/alpha/kernel/core_marvel.c b/arch/alpha/kernel/core_marvel.c index b04f1feb1dda..ced4aae8b804 100644 --- a/arch/alpha/kernel/core_marvel.c +++ b/arch/alpha/kernel/core_marvel.c @@ -660,9 +660,9 @@ __marvel_rtc_io(u8 b, unsigned long addr, int write) #ifdef CONFIG_SMP if (smp_processor_id() != boot_cpuid) - smp_call_function_on_cpu(__marvel_access_rtc, - &rtc_access, 1, 1, - cpumask_of_cpu(boot_cpuid)); + smp_call_function_single(boot_cpuid, + __marvel_access_rtc, + &rtc_access, 1, 1); else __marvel_access_rtc(&rtc_access); #else diff --git a/arch/alpha/kernel/smp.c b/arch/alpha/kernel/smp.c index 2525692db0ab..95c905be9154 100644 --- a/arch/alpha/kernel/smp.c +++ b/arch/alpha/kernel/smp.c @@ -62,6 +62,7 @@ static struct { enum ipi_message_type { IPI_RESCHEDULE, IPI_CALL_FUNC, + IPI_CALL_FUNC_SINGLE, IPI_CPU_STOP, }; @@ -558,51 +559,6 @@ send_ipi_message(cpumask_t to_whom, enum ipi_message_type operation) wripir(i); } -/* Structure and data for smp_call_function. This is designed to - minimize static memory requirements. Plus it looks cleaner. */ - -struct smp_call_struct { - void (*func) (void *info); - void *info; - long wait; - atomic_t unstarted_count; - atomic_t unfinished_count; -}; - -static struct smp_call_struct *smp_call_function_data; - -/* Atomicly drop data into a shared pointer. The pointer is free if - it is initially locked. If retry, spin until free. */ - -static int -pointer_lock (void *lock, void *data, int retry) -{ - void *old, *tmp; - - mb(); - again: - /* Compare and swap with zero. */ - asm volatile ( - "1: ldq_l %0,%1\n" - " mov %3,%2\n" - " bne %0,2f\n" - " stq_c %2,%1\n" - " beq %2,1b\n" - "2:" - : "=&r"(old), "=m"(*(void **)lock), "=&r"(tmp) - : "r"(data) - : "memory"); - - if (old == 0) - return 0; - if (! retry) - return -EBUSY; - - while (*(void **)lock) - barrier(); - goto again; -} - void handle_ipi(struct pt_regs *regs) { @@ -632,31 +588,12 @@ handle_ipi(struct pt_regs *regs) break; case IPI_CALL_FUNC: - { - struct smp_call_struct *data; - void (*func)(void *info); - void *info; - int wait; - - data = smp_call_function_data; - func = data->func; - info = data->info; - wait = data->wait; - - /* Notify the sending CPU that the data has been - received, and execution is about to begin. */ - mb(); - atomic_dec (&data->unstarted_count); - - /* At this point the structure may be gone unless - wait is true. */ - (*func)(info); - - /* Notify the sending CPU that the task is done. */ - mb(); - if (wait) atomic_dec (&data->unfinished_count); + generic_smp_call_function_interrupt(); + break; + + case IPI_CALL_FUNC_SINGLE: + generic_smp_call_function_single_interrupt(); break; - } case IPI_CPU_STOP: halt(); @@ -700,102 +637,15 @@ smp_send_stop(void) send_ipi_message(to_whom, IPI_CPU_STOP); } -/* - * Run a function on all other CPUs. - * <func> The function to run. This must be fast and non-blocking. - * <info> An arbitrary pointer to pass to the function. - * <retry> If true, keep retrying until ready. - * <wait> If true, wait until function has completed on other CPUs. - * [RETURNS] 0 on success, else a negative status code. - * - * Does not return until remote CPUs are nearly ready to execute <func> - * or are or have executed. - * You must not call this function with disabled interrupts or from a - * hardware interrupt handler or from a bottom half handler. - */ - -int -smp_call_function_on_cpu (void (*func) (void *info), void *info, int retry, - int wait, cpumask_t to_whom) +void arch_send_call_function_ipi(cpumask_t mask) { - struct smp_call_struct data; - unsigned long timeout; - int num_cpus_to_call; - - /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - data.func = func; - data.info = info; - data.wait = wait; - - cpu_clear(smp_processor_id(), to_whom); - num_cpus_to_call = cpus_weight(to_whom); - - atomic_set(&data.unstarted_count, num_cpus_to_call); - atomic_set(&data.unfinished_count, num_cpus_to_call); - - /* Acquire the smp_call_function_data mutex. */ - if (pointer_lock(&smp_call_function_data, &data, retry)) - return -EBUSY; - - /* Send a message to the requested CPUs. */ - send_ipi_message(to_whom, IPI_CALL_FUNC); - - /* Wait for a minimal response. */ - timeout = jiffies + HZ; - while (atomic_read (&data.unstarted_count) > 0 - && time_before (jiffies, timeout)) - barrier(); - - /* If there's no response yet, log a message but allow a longer - * timeout period -- if we get a response this time, log - * a message saying when we got it.. - */ - if (atomic_read(&data.unstarted_count) > 0) { - long start_time = jiffies; - printk(KERN_ERR "%s: initial timeout -- trying long wait\n", - __func__); - timeout = jiffies + 30 * HZ; - while (atomic_read(&data.unstarted_count) > 0 - && time_before(jiffies, timeout)) - barrier(); - if (atomic_read(&data.unstarted_count) <= 0) { - long delta = jiffies - start_time; - printk(KERN_ERR - "%s: response %ld.%ld seconds into long wait\n", - __func__, delta / HZ, - (100 * (delta - ((delta / HZ) * HZ))) / HZ); - } - } - - /* We either got one or timed out -- clear the lock. */ - mb(); - smp_call_function_data = NULL; - - /* - * If after both the initial and long timeout periods we still don't - * have a response, something is very wrong... - */ - BUG_ON(atomic_read (&data.unstarted_count) > 0); - - /* Wait for a complete response, if needed. */ - if (wait) { - while (atomic_read (&data.unfinished_count) > 0) - barrier(); - } - - return 0; + send_ipi_message(mask, IPI_CALL_FUNC); } -EXPORT_SYMBOL(smp_call_function_on_cpu); -int -smp_call_function (void (*func) (void *info), void *info, int retry, int wait) +void arch_send_call_function_single_ipi(int cpu) { - return smp_call_function_on_cpu (func, info, retry, wait, - cpu_online_map); + send_ipi_message(cpumask_of_cpu(cpu), IPI_CALL_FUNC_SINGLE); } -EXPORT_SYMBOL(smp_call_function); static void ipi_imb(void *ignored) diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index ca65945970f6..621b7d3b9324 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -648,6 +648,7 @@ source "kernel/time/Kconfig" config SMP bool "Symmetric Multi-Processing (EXPERIMENTAL)" depends on EXPERIMENTAL && (REALVIEW_EB_ARM11MP || MACH_REALVIEW_PB11MP) + select USE_GENERIC_SMP_HELPERS help This enables support for systems with more than one CPU. If you have a system with only one CPU, like most personal computers, say N. If diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index eefae1de334c..6344466b2113 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -68,20 +68,10 @@ enum ipi_msg_type { IPI_TIMER, IPI_RESCHEDULE, IPI_CALL_FUNC, + IPI_CALL_FUNC_SINGLE, IPI_CPU_STOP, }; -struct smp_call_struct { - void (*func)(void *info); - void *info; - int wait; - cpumask_t pending; - cpumask_t unfinished; -}; - -static struct smp_call_struct * volatile smp_call_function_data; -static DEFINE_SPINLOCK(smp_call_function_lock); - int __cpuinit __cpu_up(unsigned int cpu) { struct cpuinfo_arm *ci = &per_cpu(cpu_data, cpu); @@ -366,114 +356,15 @@ static void send_ipi_message(cpumask_t callmap, enum ipi_msg_type msg) local_irq_restore(flags); } -/* - * You must not call this function with disabled interrupts, from a - * hardware interrupt handler, nor from a bottom half handler. - */ -static int smp_call_function_on_cpu(void (*func)(void *info), void *info, - int retry, int wait, cpumask_t callmap) -{ - struct smp_call_struct data; - unsigned long timeout; - int ret = 0; - - data.func = func; - data.info = info; - data.wait = wait; - - cpu_clear(smp_processor_id(), callmap); - if (cpus_empty(callmap)) - goto out; - - data.pending = callmap; - if (wait) - data.unfinished = callmap; - - /* - * try to get the mutex on smp_call_function_data - */ - spin_lock(&smp_call_function_lock); - smp_call_function_data = &data; - - send_ipi_message(callmap, IPI_CALL_FUNC); - - timeout = jiffies + HZ; - while (!cpus_empty(data.pending) && time_before(jiffies, timeout)) - barrier(); - - /* - * did we time out? - */ - if (!cpus_empty(data.pending)) { - /* - * this may be causing our panic - report it - */ - printk(KERN_CRIT - "CPU%u: smp_call_function timeout for %p(%p)\n" - " callmap %lx pending %lx, %swait\n", - smp_processor_id(), func, info, *cpus_addr(callmap), - *cpus_addr(data.pending), wait ? "" : "no "); - - /* - * TRACE - */ - timeout = jiffies + (5 * HZ); - while (!cpus_empty(data.pending) && time_before(jiffies, timeout)) - barrier(); - - if (cpus_empty(data.pending)) - printk(KERN_CRIT " RESOLVED\n"); - else - printk(KERN_CRIT " STILL STUCK\n"); - } - - /* - * whatever happened, we're done with the data, so release it - */ - smp_call_function_data = NULL; - spin_unlock(&smp_call_function_lock); - - if (!cpus_empty(data.pending)) { - ret = -ETIMEDOUT; - goto out; - } - - if (wait) - while (!cpus_empty(data.unfinished)) - barrier(); - out: - - return 0; -} - -int smp_call_function(void (*func)(void *info), void *info, int retry, - int wait) +void arch_send_call_function_ipi(cpumask_t mask) { - return smp_call_function_on_cpu(func, info, retry, wait, - cpu_online_map); + send_ipi_message(mask, IPI_CALL_FUNC); } -EXPORT_SYMBOL_GPL(smp_call_function); -int smp_call_function_single(int cpu, void (*func)(void *info), void *info, - int retry, int wait) +void arch_send_call_function_single_ipi(int cpu) { - /* prevent preemption and reschedule on another processor */ - int current_cpu = get_cpu(); - int ret = 0; - - if (cpu == current_cpu) { - local_irq_disable(); - func(info); - local_irq_enable(); - } else - ret = smp_call_function_on_cpu(func, info, retry, wait, - cpumask_of_cpu(cpu)); - - put_cpu(); - - return ret; + send_ipi_message(cpumask_of_cpu(cpu), IPI_CALL_FUNC_SINGLE); } -EXPORT_SYMBOL_GPL(smp_call_function_single); void show_ipi_list(struct seq_file *p) { @@ -521,27 +412,6 @@ asmlinkage void __exception do_local_timer(struct pt_regs *regs) } #endif -/* - * ipi_call_function - handle IPI from smp_call_function() - * - * Note that we copy data out of the cross-call structure and then - * let the caller know that we're here and have done with their data - */ -static void ipi_call_function(unsigned int cpu) -{ - struct smp_call_struct *data = smp_call_function_data; - void (*func)(void *info) = data->func; - void *info = data->info; - int wait = data->wait; - - cpu_clear(cpu, data->pending); - - func(info); - - if (wait) - cpu_clear(cpu, data->unfinished); -} - static DEFINE_SPINLOCK(stop_lock); /* @@ -611,7 +481,11 @@ asmlinkage void __exception do_IPI(struct pt_regs *regs) break; case IPI_CALL_FUNC: - ipi_call_function(cpu); + generic_smp_call_function_interrupt(); + break; + + case IPI_CALL_FUNC_SINGLE: + generic_smp_call_function_single_interrupt(); break; case IPI_CPU_STOP: @@ -662,14 +536,13 @@ int setup_profiling_timer(unsigned int multiplier) } static int -on_each_cpu_mask(void (*func)(void *), void *info, int retry, int wait, - cpumask_t mask) +on_each_cpu_mask(void (*func)(void *), void *info, int wait, cpumask_t mask) { int ret = 0; preempt_disable(); - ret = smp_call_function_on_cpu(func, info, retry, wait, mask); + ret = smp_call_function_mask(mask, func, info, wait); if (cpu_isset(smp_processor_id(), mask)) func(info); @@ -738,7 +611,7 @@ void flush_tlb_mm(struct mm_struct *mm) { cpumask_t mask = mm->cpu_vm_mask; - on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, 1, mask); + on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, mask); } void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) @@ -749,7 +622,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) ta.ta_vma = vma; ta.ta_start = uaddr; - on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, 1, mask); + on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, mask); } void flush_tlb_kernel_page(unsigned long kaddr) @@ -771,7 +644,7 @@ void flush_tlb_range(struct vm_area_struct *vma, ta.ta_start = start; ta.ta_end = end; - on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, 1, mask); + on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, mask); } void flush_tlb_kernel_range(unsigned long start, unsigned long end) diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig index 16be41446b5b..18bcc10903b4 100644 --- a/arch/ia64/Kconfig +++ b/arch/ia64/Kconfig @@ -303,6 +303,7 @@ config VIRT_CPU_ACCOUNTING config SMP bool "Symmetric multi-processing support" + select USE_GENERIC_SMP_HELPERS help This enables support for systems with more than one CPU. If you have a system with only one CPU, say N. If you have a system with more diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c index 983296f1c813..70b7b35c224b 100644 --- a/arch/ia64/kernel/smp.c +++ b/arch/ia64/kernel/smp.c @@ -60,25 +60,9 @@ static struct local_tlb_flush_counts { static DEFINE_PER_CPU(unsigned int, shadow_flush_counts[NR_CPUS]) ____cacheline_aligned; - -/* - * Structure and data for smp_call_function(). This is designed to minimise static memory - * requirements. It also looks cleaner. - */ -static __cacheline_aligned DEFINE_SPINLOCK(call_lock); - -struct call_data_struct { - void (*func) (void *info); - void *info; - long wait; - atomic_t started; - atomic_t finished; -}; - -static volatile struct call_data_struct *call_data; - #define IPI_CALL_FUNC 0 #define IPI_CPU_STOP 1 +#define IPI_CALL_FUNC_SINGLE 2 #define IPI_KDUMP_CPU_STOP 3 /* This needs to be cacheline aligned because it is written to by *other* CPUs. */ @@ -89,38 +73,13 @@ extern void cpu_halt (void); void lock_ipi_calllock(void) { - spin_lock_irq(&call_lock); + spin_lock_irq(&call_function_lock); } void unlock_ipi_calllock(void) { - spin_unlock_irq(&call_lock); -} - -static inline void -handle_call_data(void) -{ - struct call_data_struct *data; - void (*func)(void *info); - void *info; - int wait; - - /* release the 'pointer lock' */ - data = (struct call_data_struct *)call_data; - func = data->func; - info = data->info; - wait = data->wait; - - mb(); - atomic_inc(&data->started); - /* At this point the structure may be gone unless wait is true. */ - (*func)(info); - - /* Notify the sending CPU that the task is done. */ - mb(); - if (wait) - atomic_inc(&data->finished); + spin_unlock_irq(&call_function_lock); } static void @@ -163,13 +122,15 @@ handle_IPI (int irq, void *dev_id) ops &= ~(1 << which); switch (which) { - case IPI_CALL_FUNC: - handle_call_data(); - break; - case IPI_CPU_STOP: stop_this_cpu(); break; + case IPI_CALL_FUNC: + generic_smp_call_function_interrupt(); + break; + case IPI_CALL_FUNC_SINGLE: + generic_smp_call_function_single_interrupt(); + break; #ifdef CONFIG_KEXEC case IPI_KDUMP_CPU_STOP: unw_init_running(kdump_cpu_freeze, NULL); @@ -187,6 +148,8 @@ handle_IPI (int irq, void *dev_id) return IRQ_HANDLED; } + + /* * Called with preemption disabled. */ @@ -360,190 +323,15 @@ smp_flush_tlb_mm (struct mm_struct *mm) on_each_cpu((void (*)(void *))local_finish_flush_tlb_mm, mm, 1, 1); } -/* - * Run a function on a specific CPU - * <func> The function to run. This must be fast and non-blocking. - * <info> An arbitrary pointer to pass to the function. - * <nonatomic> Currently unused. - * <wait> If true, wait until function has completed on other CPUs. - * [RETURNS] 0 on success, else a negative status code. - * - * Does not return until the remote CPU is nearly ready to execute <func> - * or is or has executed. - */ - -int -smp_call_function_single (int cpuid, void (*func) (void *info), void *info, int nonatomic, - int wait) -{ - struct call_data_struct data; - int cpus = 1; - int me = get_cpu(); /* prevent preemption and reschedule on another processor */ - - if (cpuid == me) { - local_irq_disable(); - func(info); - local_irq_enable(); - put_cpu(); - return 0; - } - - data.func = func; - data.info = info; - atomic_set(&data.started, 0); - data.wait = wait; - if (wait) - atomic_set(&data.finished, 0); - - spin_lock_bh(&call_lock); - - call_data = &data; - mb(); /* ensure store to call_data precedes setting of IPI_CALL_FUNC */ - send_IPI_single(cpuid, IPI_CALL_FUNC); - - /* Wait for response */ - while (atomic_read(&data.started) != cpus) - cpu_relax(); - - if (wait) - while (atomic_read(&data.finished) != cpus) - cpu_relax(); - call_data = NULL; - - spin_unlock_bh(&call_lock); - put_cpu(); - return 0; -} -EXPORT_SYMBOL(smp_call_function_single); - -/** - * smp_call_function_mask(): Run a function on a set of other CPUs. - * <mask> The set of cpus to run on. Must not include the current cpu. - * <func> The function to run. This must be fast and non-blocking. - * <info> An arbitrary pointer to pass to the function. - * <wait> If true, wait (atomically) until function - * has completed on other CPUs. - * - * Returns 0 on success, else a negative status code. - * - * If @wait is true, then returns once @func has returned; otherwise - * it returns just before the target cpu calls @func. - * - * You must not call this function with disabled interrupts or from a - * hardware interrupt handler or from a bottom half handler. - */ -int smp_call_function_mask(cpumask_t mask, - void (*func)(void *), void *info, - int wait) +void arch_send_call_function_single_ipi(int cpu) { - struct call_data_struct data; - cpumask_t allbutself; - int cpus; - - spin_lock(&call_lock); - allbutself = cpu_online_map; - cpu_clear(smp_processor_id(), allbutself); - - cpus_and(mask, mask, allbutself); - cpus = cpus_weight(mask); - if (!cpus) { - spin_unlock(&call_lock); - return 0; - } - - /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - data.func = func; - data.info = info; - atomic_set(&data.started, 0); - data.wait = wait; - if (wait) - atomic_set(&data.finished, 0); - - call_data = &data; - mb(); /* ensure store to call_data precedes setting of IPI_CALL_FUNC*/ - - /* Send a message to other CPUs */ - if (cpus_equal(mask, allbutself)) - send_IPI_allbutself(IPI_CALL_FUNC); - else - send_IPI_mask(mask, IPI_CALL_FUNC); - - /* Wait for response */ - while (atomic_read(&data.started) != cpus) - cpu_relax(); - - if (wait) - while (atomic_read(&data.finished) != cpus) - cpu_relax(); - call_data = NULL; - - spin_unlock(&call_lock); - return 0; - + send_IPI_single(cpu, IPI_CALL_FUNC_SINGLE); } -EXPORT_SYMBOL(smp_call_function_mask); - -/* - * this function sends a 'generic call function' IPI to all other CPUs - * in the system. - */ -/* - * [SUMMARY] Run a function on all other CPUs. - * <func> The function to run. This must be fast and non-blocking. - * <info> An arbitrary pointer to pass to the function. - * <nonatomic> currently unused. - * <wait> If true, wait (atomically) until function has completed on other CPUs. - * [RETURNS] 0 on success, else a negative status code. - * - * Does not return until remote CPUs are nearly ready to execute <func> or are or have - * executed. - * - * You must not call this function with disabled interrupts or from a - * hardware interrupt handler or from a bottom half handler. - */ -int -smp_call_function (void (*func) (void *info), void *info, int nonatomic, int wait) +void arch_send_call_function_ipi(cpumask_t mask) { - struct call_data_struct data; - int cpus; - - spin_lock(&call_lock); - cpus = num_online_cpus() - 1; - if (!cpus) { - spin_unlock(&call_lock); - return 0; - } - - /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - data.func = func; - data.info = info; - atomic_set(&data.started, 0); - data.wait = wait; - if (wait) - atomic_set(&data.finished, 0); - - call_data = &data; - mb(); /* ensure store to call_data precedes setting of IPI_CALL_FUNC */ - send_IPI_allbutself(IPI_CALL_FUNC); - - /* Wait for response */ - while (atomic_read(&data.started) != cpus) - cpu_relax(); - - if (wait) - while (atomic_read(&data.finished) != cpus) - cpu_relax(); - call_data = NULL; - - spin_unlock(&call_lock); - return 0; + send_IPI_mask(mask, IPI_CALL_FUNC); } -EXPORT_SYMBOL(smp_call_function); /* * this function calls the 'stop' function on all other CPUs in the system. diff --git a/arch/m32r/Kconfig b/arch/m32r/Kconfig index de153de2ea9f..a5f864c445b2 100644 --- a/arch/m32r/Kconfig +++ b/arch/m32r/Kconfig @@ -296,6 +296,7 @@ config PREEMPT config SMP bool "Symmetric multi-processing support" + select USE_GENERIC_SMP_HELPERS ---help--- This enables support for systems with more than one CPU. If you have a system with only one CPU, like most personal computers, say N. If diff --git a/arch/m32r/kernel/m32r_ksyms.c b/arch/m32r/kernel/m32r_ksyms.c index e6709fe950ba..16bcb189a383 100644 --- a/arch/m32r/kernel/m32r_ksyms.c +++ b/arch/m32r/kernel/m32r_ksyms.c @@ -43,9 +43,6 @@ EXPORT_SYMBOL(dcache_dummy); #endif EXPORT_SYMBOL(cpu_data); -/* Global SMP stuff */ -EXPORT_SYMBOL(smp_call_function); - /* TLB flushing */ EXPORT_SYMBOL(smp_flush_tlb_page); #endif diff --git a/arch/m32r/kernel/smp.c b/arch/m32r/kernel/smp.c index c837bc13b015..74eb7bcd5a40 100644 --- a/arch/m32r/kernel/smp.c +++ b/arch/m32r/kernel/smp.c @@ -35,22 +35,6 @@ /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ /* - * Structure and data for smp_call_function(). This is designed to minimise - * static memory requirements. It also looks cleaner. - */ -static DEFINE_SPINLOCK(call_lock); - -struct call_data_struct { - void (*func) (void *info); - void *info; - atomic_t started; - atomic_t finished; - int wait; -} __attribute__ ((__aligned__(SMP_CACHE_BYTES))); - -static struct call_data_struct *call_data; - -/* * For flush_cache_all() */ static DEFINE_SPINLOCK(flushcache_lock); @@ -96,9 +80,6 @@ void smp_invalidate_interrupt(void); void smp_send_stop(void); static void stop_this_cpu(void *); -int smp_call_function(void (*) (void *), void *, int, int); -void smp_call_function_interrupt(void); - void smp_send_timer(void); void smp_ipi_timer_interrupt(struct pt_regs *); void smp_local_timer_interrupt(void); @@ -565,86 +546,14 @@ static void stop_this_cpu(void *dummy) for ( ; ; ); } -/*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ -/* Call function Routines */ -/*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ - -/*==========================================================================* - * Name: smp_call_function - * - * Description: This routine sends a 'CALL_FUNCTION_IPI' to all other CPUs - * in the system. - * - * Born on Date: 2002.02.05 - * - * Arguments: *func - The function to run. This must be fast and - * non-blocking. - * *info - An arbitrary pointer to pass to the function. - * nonatomic - currently unused. - * wait - If true, wait (atomically) until function has - * completed on other CPUs. - * - * Returns: 0 on success, else a negative status code. Does not return - * until remote CPUs are nearly ready to execute <<func>> or - * are or have executed. - * - * Cautions: You must not call this function with disabled interrupts or - * from a hardware interrupt handler, you may call it from a - * bottom half handler. - * - * Modification log: - * Date Who Description - * ---------- --- -------------------------------------------------------- - * - *==========================================================================*/ -int smp_call_function(void (*func) (void *info), void *info, int nonatomic, - int wait) +void arch_send_call_function_ipi(cpumask_t mask) { - struct call_data_struct data; - int cpus; - -#ifdef DEBUG_SMP - unsigned long flags; - __save_flags(flags); - if (!(flags & 0x0040)) /* Interrupt Disable NONONO */ - BUG(); -#endif /* DEBUG_SMP */ - - /* Holding any lock stops cpus from going down. */ - spin_lock(&call_lock); - cpus = num_online_cpus() - 1; - - if (!cpus) { - spin_unlock(&call_lock); - return 0; - } - - /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - data.func = func; - data.info = info; - atomic_set(&data.started, 0); - data.wait = wait; - if (wait) - atomic_set(&data.finished, 0); - - call_data = &data; - mb(); - - /* Send a message to all other CPUs and wait for them to respond */ - send_IPI_allbutself(CALL_FUNCTION_IPI, 0); - - /* Wait for response */ - while (atomic_read(&data.started) != cpus) - barrier(); - - if (wait) - while (atomic_read(&data.finished) != cpus) - barrier(); - spin_unlock(&call_lock); + send_IPI_mask(mask, CALL_FUNCTION_IPI, 0); +} - return 0; +void arch_send_call_function_single_ipi(int cpu) +{ + send_IPI_mask(cpumask_of_cpu(cpu), CALL_FUNC_SINGLE_IPI, 0); } /*==========================================================================* @@ -666,27 +575,16 @@ int smp_call_function(void (*func) (void *info), void *info, int nonatomic, *==========================================================================*/ void smp_call_function_interrupt(void) { - void (*func) (void *info) = call_data->func; - void *info = call_data->info; - int wait = call_data->wait; - - /* - * Notify initiating CPU that I've grabbed the data and am - * about to execute the function - */ - mb(); - atomic_inc(&call_data->started); - /* - * At this point the info structure may be out of scope unless wait==1 - */ irq_enter(); - (*func)(info); + generic_smp_call_function_interrupt(); irq_exit(); +} - if (wait) { - mb(); - atomic_inc(&call_data->finished); - } +void smp_call_function_single_interrupt(void) +{ + irq_enter(); + generic_smp_call_function_single_interrupt(); + irq_exit(); } /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ diff --git a/arch/m32r/kernel/traps.c b/arch/m32r/kernel/traps.c index 89ba4a0b5d51..46159a4e644b 100644 --- a/arch/m32r/kernel/traps.c +++ b/arch/m32r/kernel/traps.c @@ -40,6 +40,7 @@ extern void smp_invalidate_interrupt(void); extern void smp_call_function_interrupt(void); extern void smp_ipi_timer_interrupt(void); extern void smp_flush_cache_all_interrupt(void); +extern void smp_call_function_single_interrupt(void); /* * for Boot AP function @@ -103,7 +104,7 @@ void set_eit_vector_entries(void) eit_vector[186] = (unsigned long)smp_call_function_interrupt; eit_vector[187] = (unsigned long)smp_ipi_timer_interrupt; eit_vector[188] = (unsigned long)smp_flush_cache_all_interrupt; - eit_vector[189] = 0; + eit_vector[189] = (unsigned long)smp_call_function_single_interrupt; eit_vector[190] = 0; eit_vector[191] = 0; #endif diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index e5a7c5d96364..ea70d5a225ca 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1763,6 +1763,7 @@ config SMP bool "Multi-Processing support" depends on SYS_SUPPORTS_SMP select IRQ_PER_CPU + select USE_GENERIC_SMP_HELPERS help This enables support for systems with more than one CPU. If you have a system with only one CPU, like most personal computers, say N. If diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c index 63370cdd3c90..c75b26cb61df 100644 --- a/arch/mips/kernel/smp.c +++ b/arch/mips/kernel/smp.c @@ -131,145 +131,28 @@ asmlinkage __cpuinit void start_secondary(void) cpu_idle(); } -DEFINE_SPINLOCK(smp_call_lock); - -struct call_data_struct *call_data; - -/* - * Run a function on all other CPUs. - * - * <mask> cpuset_t of all processors to run the function on. - * <func> The function to run. This must be fast and non-blocking. - * <info> An arbitrary pointer to pass to the function. - * <retry> If true, keep retrying until ready. - * <wait> If true, wait until function has completed on other CPUs. - * [RETURNS] 0 on success, else a negative status code. - * - * Does not return until remote CPUs are nearly ready to execute <func> - * or are or have executed. - * - * You must not call this function with disabled interrupts or from a - * hardware interrupt handler or from a bottom half handler: - * - * CPU A CPU B - * Disable interrupts - * smp_call_function() - * Take call_lock - * Send IPIs - * Wait for all cpus to acknowledge IPI - * CPU A has not responded, spin waiting - * for cpu A to respond, holding call_lock - * smp_call_function() - * Spin waiting for call_lock - * Deadlock Deadlock - */ -int smp_call_function_mask(cpumask_t mask, void (*func) (void *info), - void *info, int retry, int wait) +void arch_send_call_function_ipi(cpumask_t mask) { - struct call_data_struct data; - int cpu = smp_processor_id(); - int cpus; - - /* - * Can die spectacularly if this CPU isn't yet marked online - */ - BUG_ON(!cpu_online(cpu)); - - cpu_clear(cpu, mask); - cpus = cpus_weight(mask); - if (!cpus) - return 0; - - /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - data.func = func; - data.info = info; - atomic_set(&data.started, 0); - data.wait = wait; - if (wait) - atomic_set(&data.finished, 0); - - spin_lock(&smp_call_lock); - call_data = &data; - smp_mb(); - - /* Send a message to all other CPUs and wait for them to respond */ mp_ops->send_ipi_mask(mask, SMP_CALL_FUNCTION); - - /* Wait for response */ - /* FIXME: lock-up detection, backtrace on lock-up */ - while (atomic_read(&data.started) != cpus) - barrier(); - - if (wait) - while (atomic_read(&data.finished) != cpus) - barrier(); - call_data = NULL; - spin_unlock(&smp_call_lock); - - return 0; } -int smp_call_function(void (*func) (void *info), void *info, int retry, - int wait) +/* + * We reuse the same vector for the single IPI + */ +void arch_send_call_function_single_ipi(int cpu) { - return smp_call_function_mask(cpu_online_map, func, info, retry, wait); + mp_ops->send_ipi_mask(cpumask_of_cpu(cpu), SMP_CALL_FUNCTION); } +/* + * Call into both interrupt handlers, as we share the IPI for them + */ void smp_call_function_interrupt(void) { - void (*func) (void *info) = call_data->func; - void *info = call_data->info; - int wait = call_data->wait; - - /* - * Notify initiating CPU that I've grabbed the data and am - * about to execute the function. - */ - smp_mb(); - atomic_inc(&call_data->started); - - /* - * At this point the info structure may be out of scope unless wait==1. - */ irq_enter(); - (*func)(info); + generic_smp_call_function_single_interrupt(); + generic_smp_call_function_interrupt(); irq_exit(); - - if (wait) { - smp_mb(); - atomic_inc(&call_data->finished); - } -} - -int smp_call_function_single(int cpu, void (*func) (void *info), void *info, - int retry, int wait) -{ - int ret, me; - - /* - * Can die spectacularly if this CPU isn't yet marked online - */ - if (!cpu_online(cpu)) - return 0; - - me = get_cpu(); - BUG_ON(!cpu_online(me)); - - if (cpu == me) { - local_irq_disable(); - func(info); - local_irq_enable(); - put_cpu(); - return 0; - } - - ret = smp_call_function_mask(cpumask_of_cpu(cpu), func, info, retry, - wait); - - put_cpu(); - return 0; } static void stop_this_cpu(void *dummy) diff --git a/arch/mips/kernel/smtc.c b/arch/mips/kernel/smtc.c index 3e863186cd22..a516286532ab 100644 --- a/arch/mips/kernel/smtc.c +++ b/arch/mips/kernel/smtc.c @@ -877,7 +877,6 @@ static void ipi_resched_interrupt(void) /* Return from interrupt should be enough to cause scheduler check */ } - static void ipi_call_interrupt(void) { /* Invoke generic function invocation code in smp.c */ diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index bc7a19da6245..a7d4fd353c2b 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -199,6 +199,7 @@ endchoice config SMP bool "Symmetric multi-processing support" + select USE_GENERIC_SMP_HELPERS ---help--- This enables support for systems with more than one CPU. If you have a system with only one CPU, like most personal computers, say N. If diff --git a/arch/parisc/kernel/smp.c b/arch/parisc/kernel/smp.c index 85fc7754ec25..126105c76a44 100644 --- a/arch/parisc/kernel/smp.c +++ b/arch/parisc/kernel/smp.c @@ -84,19 +84,11 @@ EXPORT_SYMBOL(cpu_possible_map); DEFINE_PER_CPU(spinlock_t, ipi_lock) = SPIN_LOCK_UNLOCKED; -struct smp_call_struct { - void (*func) (void *info); - void *info; - long wait; - atomic_t unstarted_count; - atomic_t unfinished_count; -}; -static volatile struct smp_call_struct *smp_call_function_data; - enum ipi_message_type { IPI_NOP=0, IPI_RESCHEDULE=1, IPI_CALL_FUNC, + IPI_CALL_FUNC_SINGLE, IPI_CPU_START, IPI_CPU_STOP, IPI_CPU_TEST @@ -187,33 +179,12 @@ ipi_interrupt(int irq, void *dev_id) case IPI_CALL_FUNC: smp_debug(100, KERN_DEBUG "CPU%d IPI_CALL_FUNC\n", this_cpu); - { - volatile struct smp_call_struct *data; - void (*func)(void *info); - void *info; - int wait; - - data = smp_call_function_data; - func = data->func; - info = data->info; - wait = data->wait; - - mb(); - atomic_dec ((atomic_t *)&data->unstarted_count); - - /* At this point, *data can't - * be relied upon. - */ - - (*func)(info); - - /* Notify the sending CPU that the - * task is done. - */ - mb(); - if (wait) - atomic_dec ((atomic_t *)&data->unfinished_count); - } + generic_smp_call_function_interrupt(); + break; + + case IPI_CALL_FUNC_SINGLE: + smp_debug(100, KERN_DEBUG "CPU%d IPI_CALL_FUNC_SINGLE\n", this_cpu); + generic_smp_call_function_single_interrupt(); break; case IPI_CPU_START: @@ -256,6 +227,14 @@ ipi_send(int cpu, enum ipi_message_type op) spin_unlock_irqrestore(lock, flags); } +static void +send_IPI_mask(cpumask_t mask, enum ipi_message_type op) +{ + int cpu; + + for_each_cpu_mask(cpu, mask) + ipi_send(cpu, op); +} static inline void send_IPI_single(int dest_cpu, enum ipi_message_type op) @@ -295,86 +274,15 @@ smp_send_all_nop(void) send_IPI_allbutself(IPI_NOP); } - -/** - * Run a function on all other CPUs. - * <func> The function to run. This must be fast and non-blocking. - * <info> An arbitrary pointer to pass to the function. - * <retry> If true, keep retrying until ready. - * <wait> If true, wait until function has completed on other CPUs. - * [RETURNS] 0 on success, else a negative status code. - * - * Does not return until remote CPUs are nearly ready to execute <func> - * or have executed. - */ - -int -smp_call_function (void (*func) (void *info), void *info, int retry, int wait) +void arch_send_call_function_ipi(cpumask_t mask) { - struct smp_call_struct data; - unsigned long timeout; - static DEFINE_SPINLOCK(lock); - int retries = 0; - - if (num_online_cpus() < 2) - return 0; - - /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - /* can also deadlock if IPIs are disabled */ - WARN_ON((get_eiem() & (1UL<<(CPU_IRQ_MAX - IPI_IRQ))) == 0); - - - data.func = func; - data.info = info; - data.wait = wait; - atomic_set(&data.unstarted_count, num_online_cpus() - 1); - atomic_set(&data.unfinished_count, num_online_cpus() - 1); - - if (retry) { - spin_lock (&lock); - while (smp_call_function_data != 0) - barrier(); - } - else { - spin_lock (&lock); - if (smp_call_function_data) { - spin_unlock (&lock); - return -EBUSY; - } - } - - smp_call_function_data = &data; - spin_unlock (&lock); - - /* Send a message to all other CPUs and wait for them to respond */ - send_IPI_allbutself(IPI_CALL_FUNC); - - retry: - /* Wait for response */ - timeout = jiffies + HZ; - while ( (atomic_read (&data.unstarted_count) > 0) && - time_before (jiffies, timeout) ) - barrier (); - - if (atomic_read (&data.unstarted_count) > 0) { - printk(KERN_CRIT "SMP CALL FUNCTION TIMED OUT! (cpu=%d), try %d\n", - smp_processor_id(), ++retries); - goto retry; - } - /* We either got one or timed out. Release the lock */ - - mb(); - smp_call_function_data = NULL; - - while (wait && atomic_read (&data.unfinished_count) > 0) - barrier (); - - return 0; + send_IPI_mask(mask, IPI_CALL_FUNC); } -EXPORT_SYMBOL(smp_call_function); +void arch_send_call_function_single_ipi(int cpu) +{ + send_IPI_single(cpu, IPI_CALL_FUNC_SINGLE); +} /* * Flush all other CPU's tlb and then mine. Do this with on_each_cpu() diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index a7258691f00f..70931dae5fdf 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -112,6 +112,7 @@ config PPC select HAVE_KRETPROBES select HAVE_LMB select HAVE_OPROFILE + select USE_GENERIC_SMP_HELPERS if SMP config EARLY_PRINTK bool diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index ba7989ffaeee..0a8772006053 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -72,12 +72,8 @@ struct smp_ops_t *smp_ops; static volatile unsigned int cpu_callin_map[NR_CPUS]; -void smp_call_function_interrupt(void); - int smt_enabled_at_boot = 1; -static int ipi_fail_ok; - static void (*crash_ipi_function_ptr)(struct pt_regs *) = NULL; #ifdef CONFIG_PPC64 @@ -99,12 +95,15 @@ void smp_message_recv(int msg) { switch(msg) { case PPC_MSG_CALL_FUNCTION: - smp_call_function_interrupt(); + generic_smp_call_function_interrupt(); break; case PPC_MSG_RESCHEDULE: /* XXX Do we have to do this? */ set_need_resched(); break; + case PPC_MSG_CALL_FUNC_SINGLE: + generic_smp_call_function_single_interrupt(); + break; case PPC_MSG_DEBUGGER_BREAK: if (crash_ipi_function_ptr) { crash_ipi_function_ptr(get_irq_regs()); @@ -128,6 +127,19 @@ void smp_send_reschedule(int cpu) smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE); } +void arch_send_call_function_single_ipi(int cpu) +{ + smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNC_SINGLE); +} + +void arch_send_call_function_ipi(cpumask_t mask) +{ + unsigned int cpu; + + for_each_cpu_mask(cpu, mask) + smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNCTION); +} + #ifdef CONFIG_DEBUGGER void smp_send_debugger_break(int cpu) { @@ -154,215 +166,9 @@ static void stop_this_cpu(void *dummy) ; } -/* - * Structure and data for smp_call_function(). This is designed to minimise - * static memory requirements. It also looks cleaner. - * Stolen from the i386 version. - */ -static __cacheline_aligned_in_smp DEFINE_SPINLOCK(call_lock); - -static struct call_data_struct { - void (*func) (void *info); - void *info; - atomic_t started; - atomic_t finished; - int wait; -} *call_data; - -/* delay of at least 8 seconds */ -#define SMP_CALL_TIMEOUT 8 - -/* - * These functions send a 'generic call function' IPI to other online - * CPUS in the system. - * - * [SUMMARY] Run a function on other CPUs. - * <func> The function to run. This must be fast and non-blocking. - * <info> An arbitrary pointer to pass to the function. - * <nonatomic> currently unused. - * <wait> If true, wait (atomically) until function has completed on other CPUs. - * [RETURNS] 0 on success, else a negative status code. Does not return until - * remote CPUs are nearly ready to execute <<func>> or are or have executed. - * <map> is a cpu map of the cpus to send IPI to. - * - * You must not call this function with disabled interrupts or from a - * hardware interrupt handler or from a bottom half handler. - */ -static int __smp_call_function_map(void (*func) (void *info), void *info, - int nonatomic, int wait, cpumask_t map) -{ - struct call_data_struct data; - int ret = -1, num_cpus; - int cpu; - u64 timeout; - - if (unlikely(smp_ops == NULL)) - return ret; - - data.func = func; - data.info = info; - atomic_set(&data.started, 0); - data.wait = wait; - if (wait) - atomic_set(&data.finished, 0); - - /* remove 'self' from the map */ - if (cpu_isset(smp_processor_id(), map)) - cpu_clear(smp_processor_id(), map); - - /* sanity check the map, remove any non-online processors. */ - cpus_and(map, map, cpu_online_map); - - num_cpus = cpus_weight(map); - if (!num_cpus) - goto done; - - call_data = &data; - smp_wmb(); - /* Send a message to all CPUs in the map */ - for_each_cpu_mask(cpu, map) - smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNCTION); - - timeout = get_tb() + (u64) SMP_CALL_TIMEOUT * tb_ticks_per_sec; - - /* Wait for indication that they have received the message */ - while (atomic_read(&data.started) != num_cpus) { - HMT_low(); - if (get_tb() >= timeout) { - printk("smp_call_function on cpu %d: other cpus not " - "responding (%d)\n", smp_processor_id(), - atomic_read(&data.started)); - if (!ipi_fail_ok) - debugger(NULL); - goto out; - } - } - - /* optionally wait for the CPUs to complete */ - if (wait) { - while (atomic_read(&data.finished) != num_cpus) { - HMT_low(); - if (get_tb() >= timeout) { - printk("smp_call_function on cpu %d: other " - "cpus not finishing (%d/%d)\n", - smp_processor_id(), - atomic_read(&data.finished), - atomic_read(&data.started)); - debugger(NULL); - goto out; - } - } - } - - done: - ret = 0; - - out: - call_data = NULL; - HMT_medium(); - return ret; -} - -static int __smp_call_function(void (*func)(void *info), void *info, - int nonatomic, int wait) -{ - int ret; - spin_lock(&call_lock); - ret =__smp_call_function_map(func, info, nonatomic, wait, - cpu_online_map); - spin_unlock(&call_lock); - return ret; -} - -int smp_call_function(void (*func) (void *info), void *info, int nonatomic, - int wait) -{ - /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - return __smp_call_function(func, info, nonatomic, wait); -} -EXPORT_SYMBOL(smp_call_function); - -int smp_call_function_single(int cpu, void (*func) (void *info), void *info, - int nonatomic, int wait) -{ - cpumask_t map = CPU_MASK_NONE; - int ret = 0; - - /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - if (!cpu_online(cpu)) - return -EINVAL; - - cpu_set(cpu, map); - if (cpu != get_cpu()) { - spin_lock(&call_lock); - ret = __smp_call_function_map(func, info, nonatomic, wait, map); - spin_unlock(&call_lock); - } else { - local_irq_disable(); - func(info); - local_irq_enable(); - } - put_cpu(); - return ret; -} -EXPORT_SYMBOL(smp_call_function_single); - void smp_send_stop(void) { - int nolock; - - /* It's OK to fail sending the IPI, since the alternative is to - * be stuck forever waiting on the other CPU to take the interrupt. - * - * It's better to at least continue and go through reboot, since this - * function is usually called at panic or reboot time in the first - * place. - */ - ipi_fail_ok = 1; - - /* Don't deadlock in case we got called through panic */ - nolock = !spin_trylock(&call_lock); - __smp_call_function_map(stop_this_cpu, NULL, 1, 0, cpu_online_map); - if (!nolock) - spin_unlock(&call_lock); -} - -void smp_call_function_interrupt(void) -{ - void (*func) (void *info); - void *info; - int wait; - - /* call_data will be NULL if the sender timed out while - * waiting on us to receive the call. - */ - if (!call_data) - return; - - func = call_data->func; - info = call_data->info; - wait = call_data->wait; - - if (!wait) - smp_mb__before_atomic_inc(); - - /* - * Notify initiating CPU that I've grabbed the data and am - * about to execute the function - */ - atomic_inc(&call_data->started); - /* - * At this point the info structure may be out of scope unless wait==1 - */ - (*func)(info); - if (wait) { - smp_mb__before_atomic_inc(); - atomic_inc(&call_data->finished); - } + smp_call_function(stop_this_cpu, NULL, 0, 0); } struct thread_info *current_set[NR_CPUS]; @@ -592,9 +398,9 @@ int __devinit start_secondary(void *unused) secondary_cpu_time_init(); - spin_lock(&call_lock); + spin_lock(&call_function_lock); cpu_set(cpu, cpu_online_map); - spin_unlock(&call_lock); + spin_unlock(&call_function_lock); local_irq_enable(); diff --git a/arch/powerpc/platforms/cell/interrupt.c b/arch/powerpc/platforms/cell/interrupt.c index 5bf7df146022..2d5bb22d6c09 100644 --- a/arch/powerpc/platforms/cell/interrupt.c +++ b/arch/powerpc/platforms/cell/interrupt.c @@ -218,6 +218,7 @@ void iic_request_IPIs(void) { iic_request_ipi(PPC_MSG_CALL_FUNCTION, "IPI-call"); iic_request_ipi(PPC_MSG_RESCHEDULE, "IPI-resched"); + iic_request_ipi(PPC_MSG_CALL_FUNC_SINGLE, "IPI-call-single"); #ifdef CONFIG_DEBUGGER iic_request_ipi(PPC_MSG_DEBUGGER_BREAK, "IPI-debug"); #endif /* CONFIG_DEBUGGER */ diff --git a/arch/powerpc/platforms/ps3/smp.c b/arch/powerpc/platforms/ps3/smp.c index f0b12f212363..a0927a3bacb7 100644 --- a/arch/powerpc/platforms/ps3/smp.c +++ b/arch/powerpc/platforms/ps3/smp.c @@ -105,9 +105,10 @@ static void __init ps3_smp_setup_cpu(int cpu) * to index needs to be setup. */ - BUILD_BUG_ON(PPC_MSG_CALL_FUNCTION != 0); - BUILD_BUG_ON(PPC_MSG_RESCHEDULE != 1); - BUILD_BUG_ON(PPC_MSG_DEBUGGER_BREAK != 3); + BUILD_BUG_ON(PPC_MSG_CALL_FUNCTION != 0); + BUILD_BUG_ON(PPC_MSG_RESCHEDULE != 1); + BUILD_BUG_ON(PPC_MSG_CALL_FUNC_SINGLE != 2); + BUILD_BUG_ON(PPC_MSG_DEBUGGER_BREAK != 3); for (i = 0; i < MSG_COUNT; i++) { result = ps3_event_receive_port_setup(cpu, &virqs[i]); diff --git a/arch/powerpc/platforms/pseries/xics.c b/arch/powerpc/platforms/pseries/xics.c index ebebc28fe895..0fc830f576f5 100644 --- a/arch/powerpc/platforms/pseries/xics.c +++ b/arch/powerpc/platforms/pseries/xics.c @@ -383,13 +383,11 @@ static irqreturn_t xics_ipi_dispatch(int cpu) mb(); smp_message_recv(PPC_MSG_RESCHEDULE); } -#if 0 - if (test_and_clear_bit(PPC_MSG_MIGRATE_TASK, + if (test_and_clear_bit(PPC_MSG_CALL_FUNC_SINGLE, &xics_ipi_message[cpu].value)) { mb(); - smp_message_recv(PPC_MSG_MIGRATE_TASK); + smp_message_recv(PPC_MSG_CALL_FUNC_SINGLE); } -#endif #if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC) if (test_and_clear_bit(PPC_MSG_DEBUGGER_BREAK, &xics_ipi_message[cpu].value)) { diff --git a/arch/powerpc/sysdev/mpic.c b/arch/powerpc/sysdev/mpic.c index 5788a6ab1254..8e3478c995ef 100644 --- a/arch/powerpc/sysdev/mpic.c +++ b/arch/powerpc/sysdev/mpic.c @@ -1496,7 +1496,7 @@ void mpic_request_ipis(void) static char *ipi_names[] = { "IPI0 (call function)", "IPI1 (reschedule)", - "IPI2 (unused)", + "IPI2 (call function single)", "IPI3 (debugger break)", }; BUG_ON(mpic == NULL); diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig index 10a0c184444e..a724077ccf35 100644 --- a/arch/sh/Kconfig +++ b/arch/sh/Kconfig @@ -706,6 +706,7 @@ config CRASH_DUMP config SMP bool "Symmetric multi-processing support" depends on SYS_SUPPORTS_SMP + select USE_GENERIC_SMP_HELPERS ---help--- This enables support for systems with more than one CPU. If you have a system with only one CPU, like most personal computers, say N. If diff --git a/arch/sh/kernel/smp.c b/arch/sh/kernel/smp.c index 5d039d168f57..2ed8dceb297b 100644 --- a/arch/sh/kernel/smp.c +++ b/arch/sh/kernel/smp.c @@ -36,13 +36,6 @@ EXPORT_SYMBOL(cpu_possible_map); cpumask_t cpu_online_map; EXPORT_SYMBOL(cpu_online_map); -static atomic_t cpus_booted = ATOMIC_INIT(0); - -/* - * Run specified function on a particular processor. - */ -void __smp_call_function(unsigned int cpu); - static inline void __init smp_store_cpu_info(unsigned int cpu) { struct sh_cpuinfo *c = cpu_data + cpu; @@ -178,42 +171,17 @@ void smp_send_stop(void) smp_call_function(stop_this_cpu, 0, 1, 0); } -struct smp_fn_call_struct smp_fn_call = { - .lock = __SPIN_LOCK_UNLOCKED(smp_fn_call.lock), - .finished = ATOMIC_INIT(0), -}; - -/* - * The caller of this wants the passed function to run on every cpu. If wait - * is set, wait until all cpus have finished the function before returning. - * The lock is here to protect the call structure. - * You must not call this function with disabled interrupts or from a - * hardware interrupt handler or from a bottom half handler. - */ -int smp_call_function(void (*func)(void *info), void *info, int retry, int wait) +void arch_send_call_function_ipi(cpumask_t mask) { - unsigned int nr_cpus = atomic_read(&cpus_booted); - int i; - - /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - spin_lock(&smp_fn_call.lock); - - atomic_set(&smp_fn_call.finished, 0); - smp_fn_call.fn = func; - smp_fn_call.data = info; - - for (i = 0; i < nr_cpus; i++) - if (i != smp_processor_id()) - plat_send_ipi(i, SMP_MSG_FUNCTION); - - if (wait) - while (atomic_read(&smp_fn_call.finished) != (nr_cpus - 1)); + int cpu; - spin_unlock(&smp_fn_call.lock); + for_each_cpu_mask(cpu, mask) + plat_send_ipi(cpu, SMP_MSG_FUNCTION); +} - return 0; +void arch_send_call_function_single_ipi(int cpu) +{ + plat_send_ipi(cpu, SMP_MSG_FUNCTION_SINGLE); } /* Not really SMP stuff ... */ diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c index fa63c68a1819..b82d017a1744 100644 --- a/arch/sparc64/kernel/smp.c +++ b/arch/sparc64/kernel/smp.c @@ -816,8 +816,9 @@ extern unsigned long xcall_call_function; * You must not call this function with disabled interrupts or from a * hardware interrupt handler or from a bottom half handler. */ -static int smp_call_function_mask(void (*func)(void *info), void *info, - int nonatomic, int wait, cpumask_t mask) +static int sparc64_smp_call_function_mask(void (*func)(void *info), void *info, + int nonatomic, int wait, + cpumask_t mask) { struct call_data_struct data; int cpus; @@ -855,8 +856,8 @@ out_unlock: int smp_call_function(void (*func)(void *info), void *info, int nonatomic, int wait) { - return smp_call_function_mask(func, info, nonatomic, wait, - cpu_online_map); + return sparc64_smp_call_function_mask(func, info, nonatomic, wait, + cpu_online_map); } void smp_call_function_client(int irq, struct pt_regs *regs) @@ -893,7 +894,7 @@ static void tsb_sync(void *info) void smp_tsb_sync(struct mm_struct *mm) { - smp_call_function_mask(tsb_sync, mm, 0, 1, mm->cpu_vm_mask); + sparc64_smp_call_function_mask(tsb_sync, mm, 0, 1, mm->cpu_vm_mask); } extern unsigned long xcall_flush_tlb_mm; diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 9e3ee5431e31..01db61fdd8bb 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -170,6 +170,7 @@ config GENERIC_PENDING_IRQ config X86_SMP bool depends on SMP && ((X86_32 && !X86_VOYAGER) || X86_64) + select USE_GENERIC_SMP_HELPERS default y config X86_32_SMP diff --git a/arch/x86/kernel/apic_32.c b/arch/x86/kernel/apic_32.c index 84ce106b33c8..8913741bc5a4 100644 --- a/arch/x86/kernel/apic_32.c +++ b/arch/x86/kernel/apic_32.c @@ -1364,6 +1364,10 @@ void __init smp_intr_init(void) /* IPI for generic function call */ alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt); + + /* IPI for single call function */ + alloc_intr_gate(CALL_FUNCTION_SINGLE_VECTOR, + call_function_single_interrupt); } #endif diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 4e354fe90f9a..e539cb4ca3b3 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -813,6 +813,9 @@ END(invalidate_interrupt\num) ENTRY(call_function_interrupt) apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt END(call_function_interrupt) +ENTRY(call_function_single_interrupt) + apicinterrupt CALL_FUNCTION_SINGLE_VECTOR,smp_call_function_single_interrupt +END(call_function_single_interrupt) ENTRY(irq_move_cleanup_interrupt) apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt END(irq_move_cleanup_interrupt) diff --git a/arch/x86/kernel/irqinit_64.c b/arch/x86/kernel/irqinit_64.c index 31f49e8f46a7..7d8b58857ecd 100644 --- a/arch/x86/kernel/irqinit_64.c +++ b/arch/x86/kernel/irqinit_64.c @@ -199,6 +199,10 @@ void __init native_init_IRQ(void) /* IPI for generic function call */ alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt); + /* IPI for generic single function call */ + alloc_intr_gate(CALL_FUNCTION_SINGLE_VECTOR, + call_function_single_interrupt); + /* Low priority IPI to cleanup after moving an irq */ set_intr_gate(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt); #endif diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index 0cb7aadc87cd..3e051ae02681 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c @@ -121,132 +121,33 @@ static void native_smp_send_reschedule(int cpu) send_IPI_mask(cpumask_of_cpu(cpu), RESCHEDULE_VECTOR); } -/* - * Structure and data for smp_call_function(). This is designed to minimise - * static memory requirements. It also looks cleaner. - */ -static DEFINE_SPINLOCK(call_lock); - -struct call_data_struct { - void (*func) (void *info); - void *info; - atomic_t started; - atomic_t finished; - int wait; -}; - void lock_ipi_call_lock(void) { - spin_lock_irq(&call_lock); + spin_lock_irq(&call_function_lock); } void unlock_ipi_call_lock(void) { - spin_unlock_irq(&call_lock); + spin_unlock_irq(&call_function_lock); } -static struct call_data_struct *call_data; - -static void __smp_call_function(void (*func) (void *info), void *info, - int nonatomic, int wait) +void native_send_call_func_single_ipi(int cpu) { - struct call_data_struct data; - int cpus = num_online_cpus() - 1; - - if (!cpus) - return; - - data.func = func; - data.info = info; - atomic_set(&data.started, 0); - data.wait = wait; - if (wait) - atomic_set(&data.finished, 0); - - call_data = &data; - mb(); - - /* Send a message to all other CPUs and wait for them to respond */ - send_IPI_allbutself(CALL_FUNCTION_VECTOR); - - /* Wait for response */ - while (atomic_read(&data.started) != cpus) - cpu_relax(); - - if (wait) - while (atomic_read(&data.finished) != cpus) - cpu_relax(); + send_IPI_mask(cpumask_of_cpu(cpu), CALL_FUNCTION_SINGLE_VECTOR); } - -/** - * smp_call_function_mask(): Run a function on a set of other CPUs. - * @mask: The set of cpus to run on. Must not include the current cpu. - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @wait: If true, wait (atomically) until function has completed on other CPUs. - * - * Returns 0 on success, else a negative status code. - * - * If @wait is true, then returns once @func has returned; otherwise - * it returns just before the target cpu calls @func. - * - * You must not call this function with disabled interrupts or from a - * hardware interrupt handler or from a bottom half handler. - */ -static int -native_smp_call_function_mask(cpumask_t mask, - void (*func)(void *), void *info, - int wait) +void native_send_call_func_ipi(cpumask_t mask) { - struct call_data_struct data; cpumask_t allbutself; - int cpus; - - /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - /* Holding any lock stops cpus from going down. */ - spin_lock(&call_lock); allbutself = cpu_online_map; cpu_clear(smp_processor_id(), allbutself); - cpus_and(mask, mask, allbutself); - cpus = cpus_weight(mask); - - if (!cpus) { - spin_unlock(&call_lock); - return 0; - } - - data.func = func; - data.info = info; - atomic_set(&data.started, 0); - data.wait = wait; - if (wait) - atomic_set(&data.finished, 0); - - call_data = &data; - wmb(); - - /* Send a message to other CPUs */ if (cpus_equal(mask, allbutself) && cpus_equal(cpu_online_map, cpu_callout_map)) send_IPI_allbutself(CALL_FUNCTION_VECTOR); else send_IPI_mask(mask, CALL_FUNCTION_VECTOR); - - /* Wait for response */ - while (atomic_read(&data.started) != cpus) - cpu_relax(); - - if (wait) - while (atomic_read(&data.finished) != cpus) - cpu_relax(); - spin_unlock(&call_lock); - - return 0; } static void stop_this_cpu(void *dummy) @@ -268,18 +169,13 @@ static void stop_this_cpu(void *dummy) static void native_smp_send_stop(void) { - int nolock; unsigned long flags; if (reboot_force) return; - /* Don't deadlock on the call lock in panic */ - nolock = !spin_trylock(&call_lock); + smp_call_function(stop_this_cpu, NULL, 0, 0); local_irq_save(flags); - __smp_call_function(stop_this_cpu, NULL, 0, 0); - if (!nolock) - spin_unlock(&call_lock); disable_local_APIC(); local_irq_restore(flags); } @@ -301,33 +197,28 @@ void smp_reschedule_interrupt(struct pt_regs *regs) void smp_call_function_interrupt(struct pt_regs *regs) { - void (*func) (void *info) = call_data->func; - void *info = call_data->info; - int wait = call_data->wait; - ack_APIC_irq(); - /* - * Notify initiating CPU that I've grabbed the data and am - * about to execute the function - */ - mb(); - atomic_inc(&call_data->started); - /* - * At this point the info structure may be out of scope unless wait==1 - */ irq_enter(); - (*func)(info); + generic_smp_call_function_interrupt(); #ifdef CONFIG_X86_32 __get_cpu_var(irq_stat).irq_call_count++; #else add_pda(irq_call_count, 1); #endif irq_exit(); +} - if (wait) { - mb(); - atomic_inc(&call_data->finished); - } +void smp_call_function_single_interrupt(void) +{ + ack_APIC_irq(); + irq_enter(); + generic_smp_call_function_single_interrupt(); +#ifdef CONFIG_X86_32 + __get_cpu_var(irq_stat).irq_call_count++; +#else + add_pda(irq_call_count, 1); +#endif + irq_exit(); } struct smp_ops smp_ops = { @@ -338,7 +229,8 @@ struct smp_ops smp_ops = { .smp_send_stop = native_smp_send_stop, .smp_send_reschedule = native_smp_send_reschedule, - .smp_call_function_mask = native_smp_call_function_mask, + + .send_call_func_ipi = native_send_call_func_ipi, + .send_call_func_single_ipi = native_send_call_func_single_ipi, }; EXPORT_SYMBOL_GPL(smp_ops); - diff --git a/arch/x86/kernel/smpcommon.c b/arch/x86/kernel/smpcommon.c index 3449064d141a..99941b37eca0 100644 --- a/arch/x86/kernel/smpcommon.c +++ b/arch/x86/kernel/smpcommon.c @@ -25,59 +25,3 @@ __cpuinit void init_gdt(int cpu) per_cpu(cpu_number, cpu) = cpu; } #endif - -/** - * smp_call_function(): Run a function on all other CPUs. - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @nonatomic: Unused. - * @wait: If true, wait (atomically) until function has completed on other CPUs. - * - * Returns 0 on success, else a negative status code. - * - * If @wait is true, then returns once @func has returned; otherwise - * it returns just before the target cpu calls @func. - * - * You must not call this function with disabled interrupts or from a - * hardware interrupt handler or from a bottom half handler. - */ -int smp_call_function(void (*func) (void *info), void *info, int nonatomic, - int wait) -{ - return smp_call_function_mask(cpu_online_map, func, info, wait); -} -EXPORT_SYMBOL(smp_call_function); - -/** - * smp_call_function_single - Run a function on a specific CPU - * @cpu: The target CPU. Cannot be the calling CPU. - * @func: The function to run. This must be fast and non-blocking. - * @info: An arbitrary pointer to pass to the function. - * @nonatomic: Unused. - * @wait: If true, wait until function has completed on other CPUs. - * - * Returns 0 on success, else a negative status code. - * - * If @wait is true, then returns once @func has returned; otherwise - * it returns just before the target cpu calls @func. - */ -int smp_call_function_single(int cpu, void (*func) (void *info), void *info, - int nonatomic, int wait) -{ - /* prevent preemption and reschedule on another processor */ - int ret; - int me = get_cpu(); - if (cpu == me) { - local_irq_disable(); - func(info); - local_irq_enable(); - put_cpu(); - return 0; - } - - ret = smp_call_function_mask(cpumask_of_cpu(cpu), func, info, wait); - - put_cpu(); - return ret; -} -EXPORT_SYMBOL(smp_call_function_single); diff --git a/arch/x86/mach-voyager/voyager_smp.c b/arch/x86/mach-voyager/voyager_smp.c index 8dedd01e909f..fe323422e71f 100644 --- a/arch/x86/mach-voyager/voyager_smp.c +++ b/arch/x86/mach-voyager/voyager_smp.c @@ -950,94 +950,24 @@ static void smp_stop_cpu_function(void *dummy) halt(); } -static DEFINE_SPINLOCK(call_lock); - -struct call_data_struct { - void (*func) (void *info); - void *info; - volatile unsigned long started; - volatile unsigned long finished; - int wait; -}; - -static struct call_data_struct *call_data; - /* execute a thread on a new CPU. The function to be called must be * previously set up. This is used to schedule a function for * execution on all CPUs - set up the function then broadcast a * function_interrupt CPI to come here on each CPU */ static void smp_call_function_interrupt(void) { - void (*func) (void *info) = call_data->func; - void *info = call_data->info; - /* must take copy of wait because call_data may be replaced - * unless the function is waiting for us to finish */ - int wait = call_data->wait; - __u8 cpu = smp_processor_id(); - - /* - * Notify initiating CPU that I've grabbed the data and am - * about to execute the function - */ - mb(); - if (!test_and_clear_bit(cpu, &call_data->started)) { - /* If the bit wasn't set, this could be a replay */ - printk(KERN_WARNING "VOYAGER SMP: CPU %d received call funtion" - " with no call pending\n", cpu); - return; - } - /* - * At this point the info structure may be out of scope unless wait==1 - */ irq_enter(); - (*func) (info); + generic_smp_call_function_interrupt(); __get_cpu_var(irq_stat).irq_call_count++; irq_exit(); - if (wait) { - mb(); - clear_bit(cpu, &call_data->finished); - } } -static int -voyager_smp_call_function_mask(cpumask_t cpumask, - void (*func) (void *info), void *info, int wait) +static void smp_call_function_single_interrupt(void) { - struct call_data_struct data; - u32 mask = cpus_addr(cpumask)[0]; - - mask &= ~(1 << smp_processor_id()); - - if (!mask) - return 0; - - /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - data.func = func; - data.info = info; - data.started = mask; - data.wait = wait; - if (wait) - data.finished = mask; - - spin_lock(&call_lock); - call_data = &data; - wmb(); - /* Send a message to all other CPUs and wait for them to respond */ - send_CPI(mask, VIC_CALL_FUNCTION_CPI); - - /* Wait for response */ - while (data.started) - barrier(); - - if (wait) - while (data.finished) - barrier(); - - spin_unlock(&call_lock); - - return 0; + irq_enter(); + generic_smp_call_function_single_interrupt(); + __get_cpu_var(irq_stat).irq_call_count++; + irq_exit(); } /* Sorry about the name. In an APIC based system, the APICs @@ -1094,6 +1024,12 @@ void smp_qic_call_function_interrupt(struct pt_regs *regs) smp_call_function_interrupt(); } +void smp_qic_call_function_single_interrupt(struct pt_regs *regs) +{ + ack_QIC_CPI(QIC_CALL_FUNCTION_SINGLE_CPI); + smp_call_function_single_interrupt(); +} + void smp_vic_cpi_interrupt(struct pt_regs *regs) { struct pt_regs *old_regs = set_irq_regs(regs); @@ -1114,6 +1050,8 @@ void smp_vic_cpi_interrupt(struct pt_regs *regs) smp_enable_irq_interrupt(); if (test_and_clear_bit(VIC_CALL_FUNCTION_CPI, &vic_cpi_mailbox[cpu])) smp_call_function_interrupt(); + if (test_and_clear_bit(VIC_CALL_FUNCTION_SINGLE_CPI, &vic_cpi_mailbox[cpu])) + smp_call_function_single_interrupt(); set_irq_regs(old_regs); } @@ -1848,5 +1786,7 @@ struct smp_ops smp_ops = { .smp_send_stop = voyager_smp_send_stop, .smp_send_reschedule = voyager_smp_send_reschedule, - .smp_call_function_mask = voyager_smp_call_function_mask, + + .send_call_func_ipi = native_send_call_func_ipi, + .send_call_func_single_ipi = native_send_call_func_single_ipi, }; diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 6e03b94c69d5..52f0cf6db93c 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -1173,7 +1173,9 @@ static const struct smp_ops xen_smp_ops __initdata = { .smp_send_stop = xen_smp_send_stop, .smp_send_reschedule = xen_smp_send_reschedule, - .smp_call_function_mask = xen_smp_call_function_mask, + + .send_call_func_ipi = xen_smp_send_call_function_ipi, + .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi, }; #endif /* CONFIG_SMP */ diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index c5b5e39ef287..42c6e358497c 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c @@ -36,27 +36,14 @@ #include "mmu.h" cpumask_t xen_cpu_initialized_map; -static DEFINE_PER_CPU(int, resched_irq) = -1; -static DEFINE_PER_CPU(int, callfunc_irq) = -1; -static DEFINE_PER_CPU(int, debug_irq) = -1; - -/* - * Structure and data for smp_call_function(). This is designed to minimise - * static memory requirements. It also looks cleaner. - */ -static DEFINE_SPINLOCK(call_lock); -struct call_data_struct { - void (*func) (void *info); - void *info; - atomic_t started; - atomic_t finished; - int wait; -}; +static DEFINE_PER_CPU(int, resched_irq); +static DEFINE_PER_CPU(int, callfunc_irq); +static DEFINE_PER_CPU(int, callfuncsingle_irq); +static DEFINE_PER_CPU(int, debug_irq) = -1; static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id); - -static struct call_data_struct *call_data; +static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id); /* * Reschedule call back. Nothing to do, @@ -128,6 +115,17 @@ static int xen_smp_intr_init(unsigned int cpu) goto fail; per_cpu(debug_irq, cpu) = rc; + callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu); + rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR, + cpu, + xen_call_function_single_interrupt, + IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING, + callfunc_name, + NULL); + if (rc < 0) + goto fail; + per_cpu(callfuncsingle_irq, cpu) = rc; + return 0; fail: @@ -137,6 +135,9 @@ static int xen_smp_intr_init(unsigned int cpu) unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL); if (per_cpu(debug_irq, cpu) >= 0) unbind_from_irqhandler(per_cpu(debug_irq, cpu), NULL); + if (per_cpu(callfuncsingle_irq, cpu) >= 0) + unbind_from_irqhandler(per_cpu(callfuncsingle_irq, cpu), NULL); + return rc; } @@ -344,7 +345,6 @@ void xen_smp_send_reschedule(int cpu) xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR); } - static void xen_send_IPI_mask(cpumask_t mask, enum ipi_vector vector) { unsigned cpu; @@ -355,83 +355,42 @@ static void xen_send_IPI_mask(cpumask_t mask, enum ipi_vector vector) xen_send_IPI_one(cpu, vector); } +void xen_smp_send_call_function_ipi(cpumask_t mask) +{ + int cpu; + + xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR); + + /* Make sure other vcpus get a chance to run if they need to. */ + for_each_cpu_mask(cpu, mask) { + if (xen_vcpu_stolen(cpu)) { + HYPERVISOR_sched_op(SCHEDOP_yield, 0); + break; + } + } +} + +void xen_smp_send_call_function_single_ipi(int cpu) +{ + xen_send_IPI_mask(cpumask_of_cpu(cpu), XEN_CALL_FUNCTION_SINGLE_VECTOR); +} + static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id) { - void (*func) (void *info) = call_data->func; - void *info = call_data->info; - int wait = call_data->wait; - - /* - * Notify initiating CPU that I've grabbed the data and am - * about to execute the function - */ - mb(); - atomic_inc(&call_data->started); - /* - * At this point the info structure may be out of scope unless wait==1 - */ irq_enter(); - (*func)(info); + generic_smp_call_function_interrupt(); __get_cpu_var(irq_stat).irq_call_count++; irq_exit(); - if (wait) { - mb(); /* commit everything before setting finished */ - atomic_inc(&call_data->finished); - } - return IRQ_HANDLED; } -int xen_smp_call_function_mask(cpumask_t mask, void (*func)(void *), - void *info, int wait) +static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id) { - struct call_data_struct data; - int cpus, cpu; - bool yield; - - /* Holding any lock stops cpus from going down. */ - spin_lock(&call_lock); - - cpu_clear(smp_processor_id(), mask); - - cpus = cpus_weight(mask); - if (!cpus) { - spin_unlock(&call_lock); - return 0; - } - - /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - data.func = func; - data.info = info; - atomic_set(&data.started, 0); - data.wait = wait; - if (wait) - atomic_set(&data.finished, 0); - - call_data = &data; - mb(); /* write everything before IPI */ - - /* Send a message to other CPUs and wait for them to respond */ - xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR); - - /* Make sure other vcpus get a chance to run if they need to. */ - yield = false; - for_each_cpu_mask_nr(cpu, mask) - if (xen_vcpu_stolen(cpu)) - yield = true; - - if (yield) - HYPERVISOR_sched_op(SCHEDOP_yield, 0); - - /* Wait for response */ - while (atomic_read(&data.started) != cpus || - (wait && atomic_read(&data.finished) != cpus)) - cpu_relax(); - - spin_unlock(&call_lock); + irq_enter(); + generic_smp_call_function_single_interrupt(); + __get_cpu_var(irq_stat).irq_call_count++; + irq_exit(); - return 0; + return IRQ_HANDLED; } diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index 9a055592a307..1574d3a84273 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h @@ -55,13 +55,8 @@ void xen_smp_cpus_done(unsigned int max_cpus); void xen_smp_send_stop(void); void xen_smp_send_reschedule(int cpu); -int xen_smp_call_function (void (*func) (void *info), void *info, int nonatomic, - int wait); -int xen_smp_call_function_single(int cpu, void (*func) (void *info), void *info, - int nonatomic, int wait); - -int xen_smp_call_function_mask(cpumask_t mask, void (*func)(void *), - void *info, int wait); +void xen_smp_send_call_function_ipi(cpumask_t mask); +void xen_smp_send_call_function_single_ipi(int cpu); extern cpumask_t xen_cpu_initialized_map; |